From 54c243a1083ac1ca99062a3676a98675242a37f8 Mon Sep 17 00:00:00 2001 From: droot Date: Wed, 14 Oct 2026 21:31:33 -0700 Subject: [PATCH 01/19] trainers: stream checkpoint files into the page cache before loading them transformers' from_pretrained memory-maps safetensors and faults them in a page at a time. On NFS that is one ~260 KiB read in flight at ~28 ms each: Qwen3-8B took 903 s to load in the trainer on Filestore Basic HDD, while vLLM 0.30 loaded the same files from the same instance in 102 s because it prefetches them into the page cache with eight threads and 16 MiB reads (runs/2026-10-15_cold_start). training.prefetch does the same before every trainer load; OPEN_RL_PREFETCH=auto|always|off and the thread and block sizes are environment knobs. The ready keys now carry a phase record (design 016, A6): process start, prefetch result, load start and end for trainers; engine build start and end for samplers. The model view shows it as ready_at and phases. --- scripts/cold_start_case.sh | 49 +++ scripts/cold_start_client.py | 104 ++++++ scripts/cold_start_fio.sh | 59 +++ scripts/cold_start_restore_labels.sh | 15 + scripts/cold_start_timeline.py | 430 ++++++++++++++++++++++ src/server/openrl_status.py | 23 +- src/server/training_requests_processor.py | 16 +- src/server/vllm_sampler.py | 17 +- src/training/automodel_worker.py | 3 + src/training/fft_trainer_worker.py | 8 + src/training/lora_trainer_worker.py | 6 + src/training/prefetch.py | 144 ++++++++ src/training/trainer_worker.py | 3 + tests/test_prefetch.py | 66 ++++ tests/test_trainer_ready.py | 8 +- 15 files changed, 943 insertions(+), 8 deletions(-) create mode 100755 scripts/cold_start_case.sh create mode 100644 scripts/cold_start_client.py create mode 100755 scripts/cold_start_fio.sh create mode 100755 scripts/cold_start_restore_labels.sh create mode 100644 scripts/cold_start_timeline.py create mode 100644 src/training/prefetch.py create mode 100644 tests/test_prefetch.py diff --git a/scripts/cold_start_case.sh b/scripts/cold_start_case.sh new file mode 100755 index 00000000..4e85ba79 --- /dev/null +++ b/scripts/cold_start_case.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# One cold-start measurement, end to end: steer placement with node labels, +# clear old workers, record, run the demo client, collect, build the timeline. +# +# scripts/cold_start_case.sh +# scripts/cold_start_case.sh runs/2026-10-15_cold_start/A-0.5b-l4-cold Qwen/Qwen2.5-0.5B lvw8 2208 +# +# Node suffixes name the nodes allowed for each role; every other node in the +# trainer or sampler pools gets openrl.io/=false for the run. Labels are +# restored to the values in /../node-labels-before.json afterwards +# only by cold_start_restore_labels.sh, so consecutive cases can reuse them. +set -euo pipefail +CASE_DIR=$1; MODEL=$2; TRAINERS=$3; SAMPLERS=$4 +NS=${OPENRL_NAMESPACE:-openrl-system} +BASE_URL=${BASE_URL:-http://127.0.0.1:9003} +cd "$(dirname "$0")/.." +mkdir -p "$CASE_DIR" + +allow() { # role, allowed suffixes + local role=$1 allowed=$2 + for node in $(kubectl get nodes -l "openrl.io/enabled=true" -o name | sed 's|node/||'); do + kubectl get node "$node" -o jsonpath="{.metadata.labels.openrl\.io/$role}" | grep -q . || continue + local want=false + for s in $allowed; do case "$node" in *"$s") want=true;; esac; done + kubectl label node "$node" "openrl.io/$role=$want" --overwrite >/dev/null + done +} +allow trainer "$TRAINERS" +allow sampler "$SAMPLERS" +kubectl get nodes -l openrl.io/enabled=true -L openrl.io/trainer,openrl.io/sampler --no-headers > "$CASE_DIR/node-labels-during.txt" + +# Fresh pods: release whatever the previous case left, wait for the pods to go. +kubectl -n "$NS" delete workloads --all --ignore-not-found >/dev/null +for _ in $(seq 1 60); do + kubectl -n "$NS" get pods --no-headers 2>/dev/null | grep -q '^orw-' || break + sleep 2 +done +# The API server forgets the owner on its next reaper sweep; give it one. +sleep 35 + +python3 scripts/cold_start_timeline.py watch --out "$CASE_DIR" +sleep 3 +set +e +uv --project examples run python scripts/cold_start_client.py base_model="$MODEL" base_url="$BASE_URL" out="$CASE_DIR/client.jsonl" > "$CASE_DIR/client.log" 2>&1 +echo "EXIT=$?" >> "$CASE_DIR/client.log" +set -e +python3 scripts/cold_start_timeline.py collect --out "$CASE_DIR" +python3 scripts/cold_start_timeline.py timeline --out "$CASE_DIR" > /dev/null +sed -n '/Critical path/,$p' "$CASE_DIR/timeline.md" diff --git a/scripts/cold_start_client.py b/scripts/cold_start_client.py new file mode 100644 index 00000000..83647e68 --- /dev/null +++ b/scripts/cold_start_client.py @@ -0,0 +1,104 @@ +"""The demo as a client sees it: one LoRA job, with a timestamp per phase. + + uv --project examples run python scripts/cold_start_client.py base_model=Qwen/Qwen2.5-0.5B out=/path/client.jsonl + +Writes one JSON line per phase boundary, with the laptop's UTC clock, so the +timeline (scripts/cold_start_timeline.py) can line the client's view up with +the cluster's records. Phases: create_model, first forward_backward (the +trainer is serving), save_weights_and_get_sampling_client, first sample (the +sampler is serving). Nothing here is tuned; it is the smallest job that +touches both workers. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from datetime import UTC, datetime +from typing import Any, cast + +import chz +import tinker +from tinker import types + +os.environ.setdefault("TINKER_API_KEY", "tml-dummy-key") +os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") +os.environ.setdefault("HF_HUB_VERBOSITY", "error") +sys.stdout.reconfigure(line_buffering=True) + + +@chz.chz +class Config: + base_model: str = "Qwen/Qwen2.5-0.5B" + base_url: str = os.getenv("TINKER_BASE_URL", os.getenv("BASE_URL", "http://127.0.0.1:9003")) + out: str = "client.jsonl" + prompt: str = "Question: What is 2 + 2?\nAnswer:" + completion: str = " 4" + rank: int = 16 + max_tokens: int = 8 + # Stop after the trainer answers; skip the sampler phases. + trainer_only: bool = False + + +class Log: + def __init__(self, path: str): + self.path = path + self.t0 = time.time() + + def mark(self, phase: str, **extra: Any) -> None: + row = {"ts": datetime.now(UTC).isoformat(timespec="milliseconds"), "t": round(time.time() - self.t0, 3), "phase": phase, **extra} + with open(self.path, "a", encoding="utf-8") as f: + f.write(json.dumps(row) + "\n") + print(f"[cold-start] {row['t']:8.3f}s {phase} {extra if extra else ''}") + + +def build_datum(tokenizer: Any, prompt: str, completion: str) -> types.Datum: + prompt_tokens = tokenizer.encode(prompt, add_special_tokens=False) + completion_tokens = tokenizer.encode(completion, add_special_tokens=False) + tokens = prompt_tokens + completion_tokens + weights = [0.0] * len(prompt_tokens) + [1.0] * len(completion_tokens) + return types.Datum( + model_input=types.ModelInput.from_ints(tokens=tokens[:-1]), + loss_fn_inputs=cast(Any, {"target_tokens": tokens[1:], "weights": weights[1:]}), + ) + + +def main(config: Config) -> None: + log = Log(config.out) + log.mark("start", base_model=config.base_model, base_url=config.base_url) + client = tinker.ServiceClient(api_key=os.getenv("TINKER_API_KEY", "tml-dummy-key"), base_url=config.base_url) + + log.mark("create_model.start") + trainer = client.create_lora_training_client(base_model=config.base_model, rank=config.rank, train_attn=True, train_mlp=True, train_unembed=False) + log.mark("create_model.done", model_id=trainer._guaranteed_model_id()) + + tokenizer = trainer.get_tokenizer() + datum = build_datum(tokenizer, config.prompt, config.completion) + log.mark("forward_backward.start") + trainer.forward_backward([datum], "cross_entropy").result() + log.mark("forward_backward.done") + trainer.optim_step(types.AdamParams(learning_rate=1e-4)).result() + log.mark("optim_step.done") + + if config.trainer_only: + log.mark("end") + return + + log.mark("save_weights_and_get_sampling_client.start") + sampler = trainer.save_weights_and_get_sampling_client() + log.mark("save_weights_and_get_sampling_client.done") + log.mark("sample.start") + result = sampler.sample( + prompt=types.ModelInput.from_ints(tokenizer.encode(config.prompt, add_special_tokens=False)), + num_samples=1, + sampling_params=types.SamplingParams(max_tokens=config.max_tokens, temperature=0.0), + ).result() + text = tokenizer.decode(result.sequences[0].tokens, skip_special_tokens=True) + log.mark("sample.done", text=text) + log.mark("end") + + +if __name__ == "__main__": + chz.nested_entrypoint(main, allow_hyphens=True) diff --git a/scripts/cold_start_fio.sh b/scripts/cold_start_fio.sh new file mode 100755 index 00000000..bd407c98 --- /dev/null +++ b/scripts/cold_start_fio.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Read benchmark of the shared volume from a GPU node, with fio, against the +# real model files. Answers "bandwidth-bound or IOPS-bound" for the weights load. +# +# scripts/cold_start_fio.sh [model-dir-under-hub] +# scripts/cold_start_fio.sh gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 runs//fio models--Qwen--Qwen3-8B +# +# Runs a pod on that node with the open-rl-shared-pvc mounted, installs fio, +# then: (1) sequential read of the largest safetensors file, 1 MiB blocks, +# O_DIRECT so the page cache does not answer; (2) 4 KiB random reads on the +# same file for 30 s, O_DIRECT; (3) the NFS client's own counters before and +# after, from /proc/self/mountstats. Output: fio JSON and mountstats in . +set -euo pipefail +NODE=$1; OUT=$2; MODEL_DIR=${3:-models--Qwen--Qwen3-8B} +NS=${OPENRL_NAMESPACE:-openrl-system} +POD=cold-start-fio +mkdir -p "$OUT" +kubectl -n "$NS" delete pod "$POD" --ignore-not-found --wait=true >/dev/null +cat </dev/null +apiVersion: v1 +kind: Pod +metadata: + name: $POD +spec: + restartPolicy: Never + nodeName: $NODE + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: fio + image: ubuntu:24.04 + command: ["bash", "-c", "apt-get update -qq >/dev/null && apt-get install -y -qq fio >/dev/null && sleep 3600"] + volumeMounts: + - name: shared + mountPath: /mnt/shared + volumes: + - name: shared + persistentVolumeClaim: + claimName: open-rl-shared-pvc +EOF +kubectl -n "$NS" wait --for=condition=Ready pod/$POD --timeout=5m >/dev/null +for _ in $(seq 1 60); do kubectl -n "$NS" exec $POD -- which fio >/dev/null 2>&1 && break; sleep 3; done +FILE=$(kubectl -n "$NS" exec $POD -- bash -c "ls -S /mnt/shared/open-rl/huggingface/hub/$MODEL_DIR/snapshots/*/*.safetensors | head -1") +echo "file: $FILE" | tee "$OUT/file.txt" +kubectl -n "$NS" exec $POD -- bash -c "ls -l $FILE; grep -A30 ' /mnt/shared ' /proc/self/mountstats" > "$OUT/mountstats-before.txt" +kubectl -n "$NS" exec $POD -- fio --name=seqread --filename="$FILE" --rw=read --bs=1M --direct=1 --ioengine=psync --numjobs=1 --runtime=60 --time_based=0 --output-format=json > "$OUT/fio-seqread-1M-direct.json" +kubectl -n "$NS" exec $POD -- fio --name=randread --filename="$FILE" --rw=randread --bs=4k --direct=1 --ioengine=psync --numjobs=1 --runtime=30 --time_based=1 --output-format=json > "$OUT/fio-randread-4k-direct.json" +kubectl -n "$NS" exec $POD -- fio --name=seqread-buffered --filename="$FILE" --rw=read --bs=1M --direct=0 --ioengine=psync --numjobs=1 --runtime=60 --time_based=0 --output-format=json > "$OUT/fio-seqread-1M-buffered.json" +kubectl -n "$NS" exec $POD -- bash -c "grep -A30 ' /mnt/shared ' /proc/self/mountstats" > "$OUT/mountstats-after.txt" +kubectl -n "$NS" delete pod "$POD" --wait=false >/dev/null +python3 - "$OUT" <<'EOF' +import json, sys, pathlib +out = pathlib.Path(sys.argv[1]) +for name in ("fio-seqread-1M-direct", "fio-randread-4k-direct", "fio-seqread-1M-buffered"): + d = json.load(open(out / f"{name}.json"))["jobs"][0]["read"] + print(f"{name:26} {d['bw_bytes']/2**20:8.1f} MiB/s {d['iops']:9.1f} IOPS mean lat {d['lat_ns']['mean']/1e6:7.2f} ms read {d['io_bytes']/2**30:.2f} GiB in {d['runtime']/1000:.1f} s") +EOF diff --git a/scripts/cold_start_restore_labels.sh b/scripts/cold_start_restore_labels.sh new file mode 100755 index 00000000..ab489f56 --- /dev/null +++ b/scripts/cold_start_restore_labels.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Put the openrl.io role labels back to what cold_start_case.sh found. +# scripts/cold_start_restore_labels.sh runs//node-labels-before.json +set -euo pipefail +python3 - "$1" <<'EOF' +import json, subprocess, sys +for node, labels in json.load(open(sys.argv[1])).items(): + for role in ("trainer", "sampler"): + key = f"openrl.io/{role}" + if key in labels: + subprocess.run(["kubectl", "label", "node", node, f"{key}={labels[key]}", "--overwrite"], check=True, capture_output=True) + else: + subprocess.run(["kubectl", "label", "node", node, f"{key}-"], check=False, capture_output=True) + print(node, {k: v for k, v in labels.items() if k.startswith("openrl.io/")}) +EOF diff --git a/scripts/cold_start_timeline.py b/scripts/cold_start_timeline.py new file mode 100644 index 00000000..3e9123ab --- /dev/null +++ b/scripts/cold_start_timeline.py @@ -0,0 +1,430 @@ +"""Measure one OpenRL cold start from primary records, and build its timeline. + +Primary records, and the clock each one carries: + + Workload and pod objects kube-apiserver clock, 1 s resolution (creationTimestamp, + condition lastTransitionTime, containerStatuses startedAt) + Kubernetes events kubelet or scheduler clock; the Pulled message carries the + pull duration kubelet measured + worker container logs containerd on the node, ns resolution; vLLM and the trainer + print their own phase timings into them + Redis ready keys first seen by a 1 s poller running inside the Redis pod, + stamped with that pod's clock + API server access log the API server pod's clock + client phases the laptop's clock (scripts/cold_start_client.py) + +All GKE nodes and the laptop run NTP; the report prints one cross-check +(container Started event versus the first log line) so a skew would show. + +Usage, one run: + + python3 scripts/cold_start_timeline.py watch --out runs// # start recorders + uv --project examples run python scripts/cold_start_client.py base_model=... out=runs///client.jsonl + python3 scripts/cold_start_timeline.py collect --out runs// # logs, events, snapshots; stops recorders + python3 scripts/cold_start_timeline.py timeline --out runs// # timeline.json + timeline.md + +Needs kubectl pointed at the cluster. Namespace: openrl-system (OPENRL_NAMESPACE). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import signal +import subprocess +import sys +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +NS = os.getenv("OPENRL_NAMESPACE", "openrl-system") + +WORKLOAD_JSONPATH = ( + '{range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.status.reason}' + '{"\\t"}{.status.claimName}{"\\t"}{.status.podName}{"\\t"}{.status.nodeName}{"\\t"}{.metadata.deletionTimestamp}{"\\n"}{end}' +) +POD_JSONPATH = ( + '{range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.nodeName}' + '{"\\t"}{.status.containerStatuses[0].state.waiting.reason}{"\\t"}{.status.containerStatuses[0].state.running.startedAt}' + '{"\\t"}{.status.containerStatuses[0].ready}{"\\t"}{.metadata.deletionTimestamp}{"\\n"}{end}' +) + + +def now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds") + + +def sh(*args: str, check: bool = True) -> str: + return subprocess.run(args, check=check, capture_output=True, text=True).stdout + + +def kubectl(*args: str, check: bool = True) -> str: + return sh("kubectl", "-n", NS, *args, check=check) + + +def stamped_follower(cmd: list[str], out: Path) -> subprocess.Popen: + """Run cmd and prefix every output line with the local receive time.""" + script = ( + "import sys,datetime\n" + "for line in sys.stdin:\n" + " sys.stdout.write(datetime.datetime.now(datetime.UTC).isoformat(timespec='milliseconds')+'\\t'+line); sys.stdout.flush()\n" + ) + f = open(out, "a", encoding="utf-8") + producer = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + consumer = subprocess.Popen([sys.executable, "-u", "-c", script], stdin=producer.stdout, stdout=f, stderr=subprocess.STDOUT, text=True) + producer.stdout.close() # type: ignore[union-attr] + consumer.producer = producer # type: ignore[attr-defined] + return consumer + + +def poll_loop(cmd_args: list[str], out: Path, every: float = 1.0) -> subprocess.Popen: + """Run a kubectl get on an interval, each sample prefixed with the local time.""" + script = ( + "import subprocess,sys,time,datetime\n" + f"args={cmd_args!r}; every={every}\n" + "while True:\n" + " t=datetime.datetime.now(datetime.UTC).isoformat(timespec='milliseconds')\n" + " r=subprocess.run(args,capture_output=True,text=True)\n" + " for line in r.stdout.splitlines():\n" + " if line.strip(): sys.stdout.write(t+'\\t'+line+'\\n')\n" + " sys.stdout.flush(); time.sleep(every)\n" + ) + f = open(out, "a", encoding="utf-8") + return subprocess.Popen([sys.executable, "-u", "-c", script], stdout=f, stderr=subprocess.STDOUT, text=True) + + +def node_images() -> dict[str, list[str]]: + data = json.loads(sh("kubectl", "get", "nodes", "-o", "json")) + out = {} + for node in data["items"]: + names = [n for img in node["status"].get("images", []) for n in img.get("names", []) if "open-rl" in n and "@sha256" not in n] + out[node["metadata"]["name"]] = sorted(set(names)) + return out + + +# -- watch ------------------------------------------------------------------ + + +def cmd_watch(out: Path) -> None: + out.mkdir(parents=True, exist_ok=True) + (out / "node-images-before.json").write_text(json.dumps(node_images(), indent=1)) + (out / "worker-image.txt").write_text(kubectl("get", "cm", "open-rl-images", "-o", "jsonpath={.data.worker}")) + (out / "watch-started.txt").write_text(now() + "\n") + procs = [ + poll_loop(["kubectl", "-n", NS, "get", "workloads", "-o", f"jsonpath={WORKLOAD_JSONPATH}"], out / "workloads.tsv"), + poll_loop(["kubectl", "-n", NS, "get", "pods", "-o", f"jsonpath={POD_JSONPATH}"], out / "pods.tsv"), + stamped_follower( + [ + "kubectl", "-n", NS, "exec", "deploy/redis-store", "--", "sh", "-c", + 'while true; do t=$(date -u +%Y-%m-%dT%H:%M:%SZ); for k in $(redis-cli --scan --pattern "open_rl:*_ready:*"); do echo "$t $k"; done; sleep 1; done', + ], + out / "redis-ready.log", + ), + stamped_follower(["kubectl", "-n", NS, "logs", "-f", "--timestamps", "--tail=0", "deploy/open-rl-api-server"], out / "api-server.log"), + stamped_follower(["kubectl", "-n", NS, "logs", "-f", "--timestamps", "--tail=0", "deploy/open-rl-scheduler"], out / "scheduler.log"), + ] + pids = [p.pid for p in procs] + [getattr(p, "producer").pid for p in procs if hasattr(p, "producer")] + (out / "watch.pids").write_text("\n".join(map(str, pids)) + "\n") + print(f"watching; recorders {pids} write into {out}") + + +def stop_watch(out: Path) -> None: + pid_file = out / "watch.pids" + if not pid_file.exists(): + return + for pid in pid_file.read_text().split(): + try: + os.kill(int(pid), signal.SIGTERM) + except ProcessLookupError: + pass + pid_file.unlink() + + +# -- collect ------------------------------------------------------------------ + + +def cmd_collect(out: Path) -> None: + (out / "collect-started.txt").write_text(now() + "\n") + pods = json.loads(kubectl("get", "pods", "-o", "json")) + workers = [p for p in pods["items"] if p["metadata"]["name"].startswith("orw-")] + (out / "pods.json").write_text(json.dumps(pods, indent=1)) + (out / "workloads.json").write_text(kubectl("get", "workloads", "-o", "json")) + (out / "events.json").write_text(kubectl("get", "events", "-o", "json")) + for pod in workers: + name = pod["metadata"]["name"] + (out / f"log-{name}.txt").write_text(kubectl("logs", "--timestamps", name, check=False)) + (out / "node-images-after.json").write_text(json.dumps(node_images(), indent=1)) + time.sleep(2) + stop_watch(out) + print(f"collected {len(workers)} worker log(s) into {out}") + + +# -- timeline ----------------------------------------------------------------- + + +def parse_ts(value: str | None) -> datetime | None: + if not value: + return None + value = value.strip() + if value.endswith("Z"): + value = value[:-1] + "+00:00" + # Container log timestamps carry nanoseconds; Python takes microseconds. + m = re.match(r"(.*\.\d{6})\d+(\+00:00)$", value) + if m: + value = m.group(1) + m.group(2) + try: + return datetime.fromisoformat(value) + except ValueError: + return None + + +def duration_s(text: str) -> float | None: + """'3m48.333s' or '19.2s' or '1.275s' from a kubelet Pulled message.""" + m = re.search(r"in ((?:\d+h)?(?:\d+m)?[\d.]+s)", text) + if not m: + return None + total, s = 0.0, m.group(1) + for num, unit in re.findall(r"([\d.]+)([hms])", s): + total += float(num) * {"h": 3600, "m": 60, "s": 1}[unit] + return total + + +SAMPLER_MARKS = [ + ("sampler.process_first_log", None), + ("sampler.weights_loaded", r"Loading weights took ([\d.]+) seconds"), + ("sampler.model_loaded", r"Model loading took .* and ([\d.]+) seconds"), + ("sampler.compile_done", r"torch\.compile took ([\d.]+) s"), + ("sampler.kv_cache_sized", r"Available KV cache memory"), + ("sampler.graphs_captured", r"Graph capturing finished in (\d+) secs"), + ("sampler.engine_init_done", r"init engine .* took ([\d.]+) s"), +] +TRAINER_MARKS = [ + ("trainer.process_first_log", None), + ("trainer.load_start", r"Loading base model"), + ("trainer.load_done", r"Successfully loaded\."), + ("trainer.processor_started", r"Training requests processor started"), +] + + +def scan_log(path: Path, marks: list[tuple[str, str | None]]) -> dict[str, dict[str, Any]]: + found: dict[str, dict[str, Any]] = {} + if not path.exists(): + return found + for line in path.read_text(errors="replace").splitlines(): + ts, _, text = line.partition(" ") + when = parse_ts(ts) + if when is None: + continue + for name, pattern in marks: + if pattern is None: + found.setdefault(name, {"ts": when, "source": "container log, first line"}) + continue + if name in found: + continue + m = re.search(pattern, text) + if m: + entry: dict[str, Any] = {"ts": when, "source": f"container log: {text.strip()[:120]}"} + if m.groups(): + entry["self_reported_s"] = float(m.group(1)) + found[name] = entry + return found + + +def first_seen(tsv: Path, key_index: int, predicate) -> dict[str, datetime]: + """First local time each key satisfied predicate, from a poll_loop file.""" + seen: dict[str, datetime] = {} + if not tsv.exists(): + return seen + for line in tsv.read_text().splitlines(): + parts = line.split("\t") + if len(parts) < 2: + continue + when = parse_ts(parts[0]) + fields = parts[1:] + if when is None or len(fields) <= key_index: + continue + key = fields[key_index] + if key and key not in seen and predicate(fields): + seen[key] = when + return seen + + +def ready_keys(path: Path) -> dict[str, datetime]: + seen: dict[str, datetime] = {} + if not path.exists(): + return seen + for line in path.read_text().splitlines(): + parts = line.split("\t", 1) + if len(parts) < 2: + continue + body = parts[1].strip().split(" ", 1) + if len(body) == 2 and body[1] not in seen: + seen[body[1]] = parse_ts(body[0]) or parse_ts(parts[0]) # type: ignore[assignment] + return seen + + +def pod_events(events: dict[str, Any], pod: str) -> list[dict[str, Any]]: + rows = [] + for e in events.get("items", []): + if e["involvedObject"].get("name") != pod: + continue + rows.append({"reason": e["reason"], "ts": parse_ts(e.get("firstTimestamp") or e.get("eventTime")), "message": e.get("message", "")}) + return sorted(rows, key=lambda r: (r["ts"] or datetime.min.replace(tzinfo=UTC))) + + +def cmd_timeline(out: Path) -> None: + client = [json.loads(l) for l in (out / "client.jsonl").read_text().splitlines() if l.strip()] if (out / "client.jsonl").exists() else [] + marks: dict[str, dict[str, Any]] = {} + for row in client: + marks[f"client.{row['phase']}"] = {"ts": parse_ts(row["ts"]), "source": "client.jsonl (laptop clock)", **{k: v for k, v in row.items() if k not in ("ts", "t", "phase")}} + + pods = json.loads((out / "pods.json").read_text()) if (out / "pods.json").exists() else {"items": []} + events = json.loads((out / "events.json").read_text()) if (out / "events.json").exists() else {"items": []} + workloads = json.loads((out / "workloads.json").read_text()) if (out / "workloads.json").exists() else {"items": []} + ready = ready_keys(out / "redis-ready.log") + wl_first_pod = first_seen(out / "workloads.tsv", 0, lambda f: len(f) > 5 and bool(f[5])) + wl_first_node = first_seen(out / "workloads.tsv", 0, lambda f: len(f) > 6 and bool(f[6])) + wl_first_seen = first_seen(out / "workloads.tsv", 0, lambda f: True) + + workers: dict[str, dict[str, Any]] = {} + for wl in workloads.get("items", []): + role = wl["spec"]["role"] + name = wl["metadata"]["name"] + w: dict[str, Any] = {"workload": name, "model_id": wl["spec"].get("modelID"), "node": wl.get("status", {}).get("nodeName"), "pod": wl.get("status", {}).get("podName"), "marks": {}} + w["marks"][f"{role}.workload_created"] = {"ts": parse_ts(wl["metadata"]["creationTimestamp"]), "source": "Workload creationTimestamp (apiserver)"} + if name in wl_first_seen: + w["marks"][f"{role}.workload_first_seen_by_poller"] = {"ts": wl_first_seen[name], "source": "workloads.tsv poller (laptop clock, 1 s)"} + if name in wl_first_pod: + w["marks"][f"{role}.workload_reports_pod"] = {"ts": wl_first_pod[name], "source": "workloads.tsv poller: status.podName set"} + if name in wl_first_node: + w["marks"][f"{role}.workload_reports_node"] = {"ts": wl_first_node[name], "source": "workloads.tsv poller: status.nodeName set"} + for cond in wl.get("status", {}).get("conditions", []): + w["marks"][f"{role}.condition_{cond['type']}_{cond['status']}"] = {"ts": parse_ts(cond.get("lastTransitionTime")), "source": f"Workload condition {cond.get('reason', '')}"} + workers[role] = w + + for pod in pods.get("items", []): + name = pod["metadata"]["name"] + if not name.startswith("orw-"): + continue + role = "sampler" if name.endswith("sampler") or "-sampler-" in name else "trainer" + w = workers.setdefault(role, {"workload": None, "pod": name, "node": pod["spec"].get("nodeName"), "marks": {}}) + w["pod"] = name + w["node"] = pod["spec"].get("nodeName") + w["marks"][f"{role}.pod_created"] = {"ts": parse_ts(pod["metadata"]["creationTimestamp"]), "source": "pod creationTimestamp (apiserver)"} + for cond in pod.get("status", {}).get("conditions", []): + if cond["type"] in ("PodScheduled", "Initialized", "ContainersReady", "Ready") and cond["status"] == "True": + w["marks"][f"{role}.pod_{cond['type']}"] = {"ts": parse_ts(cond.get("lastTransitionTime")), "source": f"pod condition {cond['type']}"} + for cs in pod.get("status", {}).get("containerStatuses", []): + running = (cs.get("state") or {}).get("running") or {} + if running.get("startedAt"): + w["marks"][f"{role}.container_started"] = {"ts": parse_ts(running["startedAt"]), "source": "containerStatuses.state.running.startedAt"} + w["image_id"] = cs.get("imageID") + for ev in pod_events(events, name): + key = f"{role}.event_{ev['reason']}" + if key in w["marks"]: + continue + entry = {"ts": ev["ts"], "source": f"event: {ev['message'][:140]}"} + if ev["reason"] == "Pulled": + entry["kubelet_pull_s"] = duration_s(ev["message"]) + entry["pulled_from_cache"] = "already present" in ev["message"] + w["marks"][key] = entry + log = out / f"log-{name}.txt" + w["marks"].update(scan_log(log, SAMPLER_MARKS if role == "sampler" else TRAINER_MARKS)) + model_id = w.get("model_id") or (workers.get("trainer", {}).get("model_id")) + for key, when in ready.items(): + if key.startswith(f"open_rl:{role}_ready:"): + w["marks"][f"{role}.ready_key"] = {"ts": when, "source": f"redis poller: {key}"} + + report = {"case": out.name, "collected": now(), "client": marks, "workers": workers} + (out / "timeline.json").write_text(json.dumps(report, indent=1, default=str)) + (out / "timeline.md").write_text(render(report)) + print((out / "timeline.md").read_text()) + + +def fmt(ts: datetime | None) -> str: + return ts.strftime("%H:%M:%S.%f")[:-3] if ts else "unmeasured" + + +def render(report: dict[str, Any]) -> str: + client = report["client"] + t0 = client.get("client.create_model.start", {}).get("ts") + lines = [f"# Cold start timeline: {report['case']}", ""] + if t0: + lines.append(f"t=0 is the client's `create_model` call at {t0.isoformat(timespec='milliseconds')} UTC. Offsets are seconds after t=0; a negative offset is before it.") + lines.append("") + for role in ("trainer", "sampler"): + w = report["workers"].get(role) + if not w: + lines += [f"## {role}", "", "no worker observed", ""] + continue + lines += [f"## {role}: `{w.get('pod')}` on `{w.get('node')}`", "", "| Mark | Clock (UTC) | t+ s | Self-reported s | Source |", "|---|---|---|---|---|"] + rows = sorted(w["marks"].items(), key=lambda kv: (kv[1].get("ts") is None, kv[1].get("ts") or datetime.max.replace(tzinfo=UTC))) + for name, m in rows: + ts = m.get("ts") + off = f"{(ts - t0).total_seconds():.1f}" if (ts and t0) else "" + self_rep = m.get("kubelet_pull_s", m.get("self_reported_s", "")) + lines.append(f"| {name} | {fmt(ts)} | {off} | {self_rep} | {m.get('source', '')} |") + lines.append("") + lines += ["## client", "", "| Phase | Clock (UTC) | t+ s |", "|---|---|---|"] + for name, m in client.items(): + ts = m.get("ts") + lines.append(f"| {name} | {fmt(ts)} | {(ts - t0).total_seconds():.1f} |" if (ts and t0) else f"| {name} | {fmt(ts)} | |") + lines += ["", "## Critical path (derived from the marks above)", ""] + lines += critical_path(report) + return "\n".join(lines) + "\n" + + +def critical_path(report: dict[str, Any]) -> list[str]: + """Consecutive phases per worker, each with its start and end mark named.""" + out = [] + chains = { + "trainer": [ + ("client.create_model.start", "trainer.pod_created", "placement: Workload to pod"), + ("trainer.pod_created", "trainer.pod_PodScheduled", "pod scheduled"), + ("trainer.pod_PodScheduled", "trainer.event_Pulled", "image pull (0 if cached)"), + ("trainer.event_Pulled", "trainer.container_started", "container create"), + ("trainer.container_started", "trainer.load_start", "process start to load start (imports)"), + ("trainer.load_start", "trainer.load_done", "weights load"), + ("trainer.load_done", "trainer.ready_key", "ready key"), + ("client.create_model.start", "client.create_model.done", "client: create_model blocked"), + ("client.forward_backward.start", "client.forward_backward.done", "client: first forward_backward"), + ], + "sampler": [ + ("client.save_weights_and_get_sampling_client.start", "sampler.pod_created", "placement: Workload to pod"), + ("sampler.pod_created", "sampler.pod_PodScheduled", "pod scheduled"), + ("sampler.pod_PodScheduled", "sampler.event_Pulled", "image pull (0 if cached)"), + ("sampler.event_Pulled", "sampler.container_started", "container create"), + ("sampler.container_started", "sampler.process_first_log", "process start to first log"), + ("sampler.process_first_log", "sampler.weights_loaded", "imports and engine setup until weights read"), + ("sampler.weights_loaded", "sampler.compile_done", "torch.compile"), + ("sampler.compile_done", "sampler.engine_init_done", "profiling, KV cache, graph capture"), + ("sampler.engine_init_done", "sampler.ready_key", "ready key"), + ("client.sample.start", "client.sample.done", "client: first sample blocked"), + ], + } + all_marks = dict(report["client"]) + for w in report["workers"].values(): + all_marks.update(w["marks"]) + for role, chain in chains.items(): + out += [f"### {role}", "", "| Phase | From | To | Seconds |", "|---|---|---|---|"] + for a, b, label in chain: + ta, tb = all_marks.get(a, {}).get("ts"), all_marks.get(b, {}).get("ts") + secs = f"{(tb - ta).total_seconds():.1f}" if (ta and tb) else "unmeasured" + out.append(f"| {label} | {a} | {b} | {secs} |") + out.append("") + return out + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("command", choices=["watch", "collect", "timeline", "stop"]) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + {"watch": cmd_watch, "collect": cmd_collect, "timeline": cmd_timeline, "stop": stop_watch}[args.command](args.out) + + +if __name__ == "__main__": + main() diff --git a/src/server/openrl_status.py b/src/server/openrl_status.py index 16acf057..300938f5 100644 --- a/src/server/openrl_status.py +++ b/src/server/openrl_status.py @@ -14,6 +14,7 @@ """ import asyncio +import json from dataclasses import dataclass from typing import Any @@ -35,6 +36,19 @@ class StatusContext: worker_launcher: str # as get_server_capabilities reports it +def ready_record(raw: str | None) -> dict[str, Any] | None: + """The ready key's value: None when the worker has not announced itself, a + dict otherwise. Workers before v0.0.8 wrote "1"; the view treats that as + announced with no phase record.""" + if raw is None: + return None + try: + parsed = json.loads(raw) + except ValueError: + return {} + return parsed if isinstance(parsed, dict) else {} + + def describe_worker(worker: dict[str, Any]) -> str: """One line for a human or an example's progress output.""" age = f", {int(worker['age_seconds'])}s" if worker.get("age_seconds") is not None else "" @@ -56,10 +70,15 @@ async def workers_of_owner(ctx: StatusContext, owner: str) -> tuple[list[dict[st worker = {field: found.get(field) for field in WORKER_FIELDS} # The launcher sees a pod run minutes before the model is loaded; each # worker announces itself when it can serve (open_rl:_ready:). - announced = await ctx.state.get_value(f"open_rl:{found['role']}_ready:{found['model_id']}") is not None - if worker["state"] == READY and not announced: + record = ready_record(await ctx.state.get_value(f"open_rl:{found['role']}_ready:{found['model_id']}")) + if worker["state"] == READY and record is None: worker["state"] = STARTING worker["detail"] = f"{worker['detail']}, loading the model" + if record: + # When the worker became ready and how it got there (design 016, A6): + # process start, prefetch, load start and end, as the worker recorded them. + worker["ready_at"] = record.get("ready_at") + worker["phases"] = record.get("phases") or {} worker["summary"] = describe_worker(worker) out.append(worker) return out, None diff --git a/src/server/training_requests_processor.py b/src/server/training_requests_processor.py index 70fb1cf5..4c1dc76b 100644 --- a/src/server/training_requests_processor.py +++ b/src/server/training_requests_processor.py @@ -37,6 +37,8 @@ # The trainer's readiness key, the counterpart of the sampler's. A long-lived # shared LoRA trainer outlives the TTL, so the loop refreshes it. READY_TTL_SECONDS = 3600 +# When this process started, for the phase record the ready key carries. +PROCESS_STARTED_AT = time.time() READY_REFRESH_SECONDS = 60 @@ -116,6 +118,7 @@ def __init__( self.snapshot_registered = False self.state: StateStore | None = None # opened on first use; tests inject one self.ready_announced_at = float("-inf") + self.ready_at: float | None = None async def run(self) -> None: print(f"[WORKER] Training requests processor started (model={self.model_id} active_set={self.active_tenant_set_id}).") @@ -160,9 +163,20 @@ async def announce_readiness(self) -> None: return if self.state is None: self.state = get_state_store() - await self.state.set_value(key, "1", ttl_seconds=READY_TTL_SECONDS) + if self.ready_at is None: + self.ready_at = time.time() + await self.state.set_value(key, json.dumps(self.ready_record()), ttl_seconds=READY_TTL_SECONDS) self.ready_announced_at = time.monotonic() + def ready_record(self) -> dict[str, Any]: + """What the key says: when this worker became ready and how it got there. + Timestamps are Unix seconds from the worker's clock; `prefetch` is what + training.prefetch did before the load, or null. The model view shows it so a + slow start can be attributed without the worker's log.""" + phases: dict[str, Any] = {"process_start": PROCESS_STARTED_AT} + phases.update(getattr(self.worker, "load_phases", None) or {}) + return {"ready_at": self.ready_at, "phases": phases} + async def withdraw_readiness(self) -> None: key = self.ready_key() if key is not None and self.state is not None: diff --git a/src/server/vllm_sampler.py b/src/server/vllm_sampler.py index bc0dba96..84b461f0 100644 --- a/src/server/vllm_sampler.py +++ b/src/server/vllm_sampler.py @@ -4,6 +4,7 @@ import argparse import asyncio import hashlib +import json import os import signal import time @@ -32,6 +33,7 @@ SHUTDOWN_SENTINEL = "SHUTDOWN_SENTINEL" TMP_DIR = os.getenv("OPEN_RL_TMP_DIR", "/tmp/open-rl") READY_TTL_SECONDS = 3600 +PROCESS_STARTED_AT = time.time() ENGINE_POLL_SECONDS = 5 @@ -274,9 +276,11 @@ async def process_request(sampler: Sampler, store: RequestStore, request: dict[s await store.set_future(request["request_id"], result) -async def mark_ready(state: StateStore, model_id: str) -> None: - """The gateway waits on this key. A shared LoRA sampler outlives the TTL, so the loop refreshes it.""" - await state.set_value(f"open_rl:sampler_ready:{model_id}", "1", ttl_seconds=READY_TTL_SECONDS) +async def mark_ready(state: StateStore, model_id: str, record: dict[str, Any] | None = None) -> None: + """The gateway waits on this key. A shared LoRA sampler outlives the TTL, so the loop refreshes it. + The value records when the sampler became ready and how long the engine took to build + (design 016, A6); vLLM's own load, compile and capture timings stay in its log.""" + await state.set_value(f"open_rl:sampler_ready:{model_id}", json.dumps(record or {"ready_at": time.time()}), ttl_seconds=READY_TTL_SECONDS) async def serve( @@ -291,11 +295,16 @@ async def serve( Queue and store errors are retried. A dead engine or a process the time slicer could not park raises, so the worker exits and is restarted. """ + engine_build_start = time.time() async with holding_gpu(time_slicer, workload): sampler = Sampler(make_engine()) if time_slicer is not None: await sampler.sleep() # give the memory back before releasing the slot state = get_state_store() + record = { + "ready_at": time.time(), + "phases": {"process_start": PROCESS_STARTED_AT, "engine_build_start": engine_build_start, "engine_ready": time.time()}, + } try: try: ready_at = float("-inf") @@ -306,7 +315,7 @@ async def serve( started = False try: if time.monotonic() - ready_at > 60: - await mark_ready(state, model_id) + await mark_ready(state, model_id, record) ready_at = time.monotonic() batch = await store.get_sampling_requests_for_model(model_id) if not batch: diff --git a/src/training/automodel_worker.py b/src/training/automodel_worker.py index 29a678be..abaf6e22 100644 --- a/src/training/automodel_worker.py +++ b/src/training/automodel_worker.py @@ -41,6 +41,7 @@ from transformers import AutoConfig, AutoTokenizer from training.distributed import is_primary +from training.prefetch import prefetch from training.trainer_worker import BaseTrainerWorker, Datum from training.types import FFTConfig, LoraConfig @@ -313,7 +314,9 @@ def load_model(self, base_model_name: str) -> None: if self.cp_size > 1: kwargs["attn_implementation"] = "sdpa" # from_pretrained applies LoRA, loads the base weights and freezes all but the adapters. + self.load_phases = {"load_start": time.time(), "prefetch": prefetch(base_model_name)} self.model = NeMoAutoModelForCausalLM.from_pretrained(base_model_name, distributed_setup=self.build_distributed_setup(), **kwargs) + self.load_phases["load_done"] = time.time() if RECOMPUTE_NUM_LAYERS > 0: install_group_checkpointing(self.model, RECOMPUTE_NUM_LAYERS) print(f"Loaded Automodel {base_model_name} with LoRA rank {MAX_LORA_RANK}.") diff --git a/src/training/fft_trainer_worker.py b/src/training/fft_trainer_worker.py index c547ab09..a6d01d95 100644 --- a/src/training/fft_trainer_worker.py +++ b/src/training/fft_trainer_worker.py @@ -15,6 +15,7 @@ import torch from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedModel +from training.prefetch import prefetch from training.trainer_worker import BaseTrainerWorker, Datum from training.types import FFTConfig @@ -77,11 +78,15 @@ def load_base_model(self, base_model_name: str) -> None: num_gpus = torch.cuda.device_count() if torch.cuda.is_available() else 0 target_device = "auto" if num_gpus > 1 else self.device print(f"Loading full fine-tuning model {base_model_name} (target device map: {target_device}, visible GPUs: {num_gpus})...") + self.load_phases = {"load_start": time.time()} self.base_model_name = base_model_name self.tokenizer = AutoTokenizer.from_pretrained(base_model_name) dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + # See training.prefetch: mmap reads over NFS are page-sized and serial. + self.load_phases["prefetch"] = prefetch(base_model_name) self.model = AutoModelForCausalLM.from_pretrained(base_model_name, dtype=dtype, device_map=target_device) + self.load_phases["load_done"] = time.time() print("Successfully loaded full fine-tuning model.") def create_model(self, base_model_name: str, model_id: str | None = None, config: FFTConfig | None = None) -> None: @@ -301,11 +306,14 @@ def load_from_state(self, model_id: str, state_path: str, restore_optimizer: boo raise ValueError(f"metadata.json at {state_path} missing base_model") self.base_model_name = base_model + self.load_phases = {"load_start": time.time()} self.tokenizer = AutoTokenizer.from_pretrained(state_path) dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 num_gpus = torch.cuda.device_count() if torch.cuda.is_available() else 0 target_device = "auto" if num_gpus > 1 else self.device + self.load_phases["prefetch"] = prefetch(state_path) self.model = AutoModelForCausalLM.from_pretrained(state_path, dtype=dtype, device_map=target_device) + self.load_phases["load_done"] = time.time() self.prepare_model_for_training() if restore_optimizer and metadata.get("has_optimizer"): diff --git a/src/training/lora_trainer_worker.py b/src/training/lora_trainer_worker.py index 83561e5e..f55336ae 100644 --- a/src/training/lora_trainer_worker.py +++ b/src/training/lora_trainer_worker.py @@ -13,6 +13,7 @@ from peft import PeftModelForCausalLM, get_peft_model from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedModel +from training.prefetch import prefetch from training.trainer_worker import BaseTrainerWorker, Datum from training.types import LoraConfig @@ -46,11 +47,16 @@ def load_base_model(self, base_model_name: str) -> None: return print(f"Loading base model {base_model_name} to {self.device}...") + self.load_phases = {"load_start": time.time()} self.base_model_name = base_model_name self.tokenizer = AutoTokenizer.from_pretrained(base_model_name) dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + # from_pretrained memory-maps the shards and reads them a page at a time; on + # a network filesystem that is minutes. Stream them into the page cache first. + self.load_phases["prefetch"] = prefetch(base_model_name) self.base_model = AutoModelForCausalLM.from_pretrained(base_model_name, dtype=dtype, device_map=self.device) + self.load_phases["load_done"] = time.time() print("Successfully loaded.") def target_lora_modules(self, config: LoraConfig) -> list[str]: diff --git a/src/training/prefetch.py b/src/training/prefetch.py new file mode 100644 index 00000000..503891f3 --- /dev/null +++ b/src/training/prefetch.py @@ -0,0 +1,144 @@ +"""Read a checkpoint's files into the page cache before a loader maps them. + +Why: transformers' from_pretrained memory-maps safetensors and touches them +tensor by tensor. On NFS that is one synchronous read of a few hundred KiB at +a time, each paying the server's latency. Measured on Filestore Basic HDD +(runs/2026-10-15_cold_start): Qwen3-8B took 903 s in the trainer at 260 KiB +per read and 28 ms per read, about two in flight. vLLM 0.30 loads the same +files from the same instance in 102 s because it first streams them into the +page cache with eight threads and 16 MiB reads. This module does the same for +the trainers, so the mmap that follows hits memory. + +Controlled by the environment, so a deployment can turn it off or tune it: + + OPEN_RL_PREFETCH auto (default: only on network filesystems), always, off + OPEN_RL_PREFETCH_THREADS 8 + OPEN_RL_PREFETCH_BLOCK_MIB 16 + +The prefetch is skipped when the files would not fit in 90% of the memory +available to this process, because the page cache would evict them before +the loader arrives. +""" + +from __future__ import annotations + +import os +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +PATTERNS = ("*.safetensors", "*.bin", "*.pt", "*.pth") +NETWORK_FS = {"nfs", "nfs4", "lustre", "fuse", "fuse.gcsfuse", "cifs", "smb3"} +# One task per this many bytes, so a few large shards still spread over all threads. +CHUNK_BYTES = 256 * 1024 * 1024 + + +def checkpoint_dir(name_or_path: str) -> Path | None: + """The directory holding the checkpoint: the path itself, or the Hugging Face + cache snapshot for a model id. No network: a model that is not cached yet is + downloaded by the loader, not by us.""" + path = Path(name_or_path) + if path.is_dir(): + return path + try: + from huggingface_hub import snapshot_download + + return Path(snapshot_download(name_or_path, local_files_only=True, allow_patterns=list(PATTERNS))) + except Exception: + return None + + +def checkpoint_files(name_or_path: str) -> list[Path]: + directory = checkpoint_dir(name_or_path) + if directory is None: + return [] + files = [p for pattern in PATTERNS for p in directory.glob(pattern) if p.is_file()] + # The HF cache keeps symlinks into blobs/; stat follows them, which is what we want. + return sorted(set(files), key=lambda p: p.stat().st_size, reverse=True) + + +def fs_type(path: Path) -> str: + """The filesystem type of the mount that holds path, from /proc/mounts.""" + best, best_type = "", "" + try: + resolved = str(path.resolve()) + with open("/proc/mounts") as f: + for line in f: + parts = line.split() + if len(parts) < 3: + continue + mount = parts[1] + if (resolved == mount or resolved.startswith(mount.rstrip("/") + "/")) and len(mount) >= len(best): + best, best_type = mount, parts[2] + except OSError: + return "" + return best_type + + +def available_bytes() -> int: + """What the page cache can hold for us: MemAvailable, capped by the cgroup limit.""" + avail = 0 + try: + with open("/proc/meminfo") as f: + for line in f: + if line.startswith("MemAvailable:"): + avail = int(line.split()[1]) * 1024 + except OSError: + return 0 + for limit_file in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"): + try: + raw = Path(limit_file).read_text().strip() + if raw.isdigit(): + avail = min(avail, int(raw)) + except OSError: + continue + return avail + + +def read_range(path: Path, offset: int, length: int, block: int) -> int: + """Read [offset, offset+length) with plain reads; the kernel keeps the pages.""" + done = 0 + fd = os.open(path, os.O_RDONLY) + try: + while done < length: + chunk = os.pread(fd, min(block, length - done), offset + done) + if not chunk: + break + done += len(chunk) + finally: + os.close(fd) + return done + + +def prefetch(name_or_path: str, log=print) -> dict[str, Any] | None: + """Warm the page cache for name_or_path's checkpoint files. Returns what it + did, for the ready key's phase record, or None when it did nothing.""" + mode = os.getenv("OPEN_RL_PREFETCH", "auto").lower() + if mode == "off": + return None + files = checkpoint_files(name_or_path) + if not files: + return None + total = sum(p.stat().st_size for p in files) + kind = fs_type(files[0]) + if mode == "auto" and kind not in NETWORK_FS: + return None + avail = available_bytes() + if avail and total > 0.9 * avail and mode != "always": + log(f"[prefetch] {total / 2**30:.1f} GiB of checkpoint files would not fit in {avail / 2**30:.1f} GiB of available memory; not prefetching") + return None + threads = int(os.getenv("OPEN_RL_PREFETCH_THREADS", "8")) + block = int(os.getenv("OPEN_RL_PREFETCH_BLOCK_MIB", "16")) * 1024 * 1024 + tasks = [] + for path in files: + size = path.stat().st_size + for offset in range(0, size, CHUNK_BYTES): + tasks.append((path, offset, min(CHUNK_BYTES, size - offset))) + log(f"[prefetch] reading {len(files)} file(s), {total / 2**30:.2f} GiB, from {kind or 'unknown fs'} into the page cache with {threads} threads") + started = time.time() + with ThreadPoolExecutor(max_workers=threads) as pool: + read = sum(pool.map(lambda t: read_range(t[0], t[1], t[2], block), tasks)) + seconds = time.time() - started + log(f"[prefetch] done: {read / 2**30:.2f} GiB in {seconds:.1f} s ({read / 2**20 / max(seconds, 1e-6):.0f} MiB/s)") + return {"files": len(files), "bytes": read, "seconds": round(seconds, 2), "fs": kind, "threads": threads, "started_at": started} diff --git a/src/training/trainer_worker.py b/src/training/trainer_worker.py index 046b295e..0178cde1 100644 --- a/src/training/trainer_worker.py +++ b/src/training/trainer_worker.py @@ -27,6 +27,9 @@ class BaseTrainerWorker: def __init__(self): self.tokenizer: PreTrainedTokenizerBase | None = None + # When the model load started and finished, and what the page-cache + # prefetch did; the request processor publishes it with the ready key. + self.load_phases: dict[str, Any] = {} if torch.cuda.is_available(): self.device = torch.device("cuda", local_rank()) diff --git a/tests/test_prefetch.py b/tests/test_prefetch.py new file mode 100644 index 00000000..a3df9ec1 --- /dev/null +++ b/tests/test_prefetch.py @@ -0,0 +1,66 @@ +"""training.prefetch warms the page cache for a checkpoint before the loader +maps it. These tests cover its decisions; the effect on load time is measured +in runs/2026-10-15_cold_start.""" + +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from training import prefetch + + +class PrefetchTest(unittest.TestCase): + def setUp(self): + self.dir = Path(self.enterContext(tempfile.TemporaryDirectory())) + (self.dir / "model-00001-of-00002.safetensors").write_bytes(os.urandom(3 * 1024 * 1024)) + (self.dir / "model-00002-of-00002.safetensors").write_bytes(os.urandom(1024 * 1024 + 17)) + (self.dir / "config.json").write_text("{}") + + def test_lists_checkpoint_files_largest_first(self): + files = prefetch.checkpoint_files(str(self.dir)) + self.assertEqual([f.name for f in files], ["model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"]) + + def test_an_uncached_model_id_is_left_to_the_loader(self): + with patch.dict(os.environ, {"HF_HOME": str(self.dir / "empty-hub")}): + self.assertEqual(prefetch.checkpoint_files("org/not-downloaded"), []) + + def test_auto_mode_skips_local_filesystems(self): + with patch.dict(os.environ, {"OPEN_RL_PREFETCH": "auto"}), patch.object(prefetch, "fs_type", return_value="ext4"): + self.assertIsNone(prefetch.prefetch(str(self.dir), log=lambda *_: None)) + + def test_auto_mode_reads_everything_on_a_network_filesystem(self): + with patch.dict(os.environ, {"OPEN_RL_PREFETCH": "auto", "OPEN_RL_PREFETCH_THREADS": "3"}), patch.object(prefetch, "fs_type", return_value="nfs"): + result = prefetch.prefetch(str(self.dir), log=lambda *_: None) + self.assertEqual(result["bytes"], 4 * 1024 * 1024 + 17) + self.assertEqual((result["files"], result["threads"], result["fs"]), (2, 3, "nfs")) + + def test_always_reads_even_on_local_disk_and_off_never_does(self): + with patch.dict(os.environ, {"OPEN_RL_PREFETCH": "always"}), patch.object(prefetch, "fs_type", return_value="ext4"): + self.assertEqual(prefetch.prefetch(str(self.dir), log=lambda *_: None)["files"], 2) + with patch.dict(os.environ, {"OPEN_RL_PREFETCH": "off"}), patch.object(prefetch, "fs_type", return_value="nfs"): + self.assertIsNone(prefetch.prefetch(str(self.dir))) + + def test_skips_when_the_files_would_not_fit_in_memory(self): + messages = [] + with ( + patch.dict(os.environ, {"OPEN_RL_PREFETCH": "auto"}), + patch.object(prefetch, "fs_type", return_value="nfs"), + patch.object(prefetch, "available_bytes", return_value=1024 * 1024), + ): + self.assertIsNone(prefetch.prefetch(str(self.dir), log=messages.append)) + self.assertIn("not prefetching", messages[0]) + + def test_large_files_are_split_across_threads(self): + with ( + patch.dict(os.environ, {"OPEN_RL_PREFETCH": "always", "OPEN_RL_PREFETCH_BLOCK_MIB": "1"}), + patch.object(prefetch, "CHUNK_BYTES", 1024 * 1024), + patch.object(prefetch, "fs_type", return_value="nfs"), + ): + result = prefetch.prefetch(str(self.dir), log=lambda *_: None) + self.assertEqual(result["bytes"], 4 * 1024 * 1024 + 17) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_trainer_ready.py b/tests/test_trainer_ready.py index cad321fc..0991e84f 100644 --- a/tests/test_trainer_ready.py +++ b/tests/test_trainer_ready.py @@ -3,6 +3,7 @@ while the loop runs, deleted on exit. The model view turns it into `ready`.""" import asyncio +import json import os import unittest from unittest.mock import patch @@ -35,8 +36,13 @@ def test_announces_only_once_a_model_is_loaded_and_withdraws_on_exit(self): asyncio.run(self.proc.announce_readiness()) self.assertIsNone(self.key(), "no model yet: the view must keep saying starting") self.worker.loaded = True + self.worker.load_phases = {"load_start": 100.0, "load_done": 160.0, "prefetch": {"bytes": 7, "seconds": 1.5}} asyncio.run(self.proc.announce_readiness()) - self.assertEqual(self.key(), "1") + record = json.loads(self.key()) + self.assertGreater(record["ready_at"], 0) + self.assertIn("process_start", record["phases"]) + self.assertEqual(record["phases"]["load_done"], 160.0) + self.assertEqual(record["phases"]["prefetch"]["bytes"], 7) asyncio.run(self.proc.withdraw_readiness()) self.assertIsNone(self.key()) From 18409bd3d71bd660cb56cef6d4bee32fddd77ed2 Mon Sep 17 00:00:00 2001 From: droot Date: Wed, 14 Oct 2026 22:23:50 -0700 Subject: [PATCH 02/19] runs: cold start on open-rl-dra, measured, before and after the trainer prefetch Measurement tooling: scripts/cold_start_client.py stamps the demo's client phases; cold_start_timeline.py records Workloads, pods, events, worker logs, the Redis ready keys and the API server log and joins them into a timeline; cold_start_case.sh runs one case; cold_start_fio.sh, cold_start_monitoring.py and cold_start_resident.py check the storage from the node, Cloud Monitoring and the page cache. runs/2026-10-15_cold_start holds the records and REPORT.md. Headline: the 8B trainer load took 903 s on a cold Filestore because from_pretrained reads 260 KiB at a time with two in flight; vLLM read the same files in 102 s. Latency-bound, not bandwidth-bound. With the prefetch the trainer loads in 12-13.5 s on a warm server cache (39.9 s before) and reads cold disk at 115 MiB/s (10.8-17 before). --- .../A-0.5b-l4-cold.run.log | 34 + .../A-0.5b-l4-cold/api-server.log | 71 + .../A-0.5b-l4-cold/client.jsonl | 11 + .../A-0.5b-l4-cold/client.log | 12 + .../A-0.5b-l4-cold/collect-started.txt | 1 + .../A-0.5b-l4-cold/events.json | 661 ++ ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 + ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 46 + .../A-0.5b-l4-cold/monitoring.json | 329 + .../A-0.5b-l4-cold/monitoring.md | 12 + .../A-0.5b-l4-cold/node-images-after.json | 48 + .../A-0.5b-l4-cold/node-images-before.json | 50 + .../A-0.5b-l4-cold/node-labels-during.txt | 8 + .../A-0.5b-l4-cold/pods.json | 6782 ++++++++++++++++ .../A-0.5b-l4-cold/redis-ready.log | 299 + .../A-0.5b-l4-cold/scheduler.log | 2 + .../A-0.5b-l4-cold/timeline.json | 275 + .../A-0.5b-l4-cold/timeline.md | 111 + .../A-0.5b-l4-cold/watch-started.txt | 1 + .../A-0.5b-l4-cold/worker-image.txt | 1 + .../A-0.5b-l4-cold/workloads.json | 355 + .../B-0.5b-l4-cached/api-server.log | 31 + .../B-0.5b-l4-cached/client.jsonl | 11 + .../B-0.5b-l4-cached/client.log | 16 + .../B-0.5b-l4-cached/collect-started.txt | 1 + .../B-0.5b-l4-cached/events.json | 269 + ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 + ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 47 + .../B-0.5b-l4-cached/monitoring.json | 212 + .../B-0.5b-l4-cached/monitoring.md | 7 + .../B-0.5b-l4-cached/node-images-after.json | 50 + .../B-0.5b-l4-cached/node-images-before.json | 50 + .../B-0.5b-l4-cached/pods.json | 6782 ++++++++++++++++ .../B-0.5b-l4-cached/redis-ready.log | 136 + .../B-0.5b-l4-cached/scheduler.log | 2 + .../B-0.5b-l4-cached/timeline.json | 267 + .../B-0.5b-l4-cached/timeline.md | 109 + .../B-0.5b-l4-cached/watch-started.txt | 1 + .../B-0.5b-l4-cached/worker-image.txt | 1 + .../B-0.5b-l4-cached/workloads.json | 355 + .../C-8b-h100-cold.run.log | 34 + .../C-8b-h100-cold/api-server.log | 176 + .../C-8b-h100-cold/client.jsonl | 11 + .../C-8b-h100-cold/client.log | 12 + .../C-8b-h100-cold/collect-started.txt | 1 + .../C-8b-h100-cold/events.json | 1053 +++ .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 223 + .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 220 + .../C-8b-h100-cold/monitoring.json | 725 ++ .../C-8b-h100-cold/monitoring.md | 26 + .../C-8b-h100-cold/node-images-after.json | 50 + .../C-8b-h100-cold/node-images-before.json | 48 + .../C-8b-h100-cold/node-labels-during.txt | 8 + .../C-8b-h100-cold/pods.json | 6782 ++++++++++++++++ .../C-8b-h100-cold/redis-ready.log | 1734 +++++ .../sampler-mountstats-sample1.txt | 2 + .../sampler-mountstats-sample2.txt | 2 + .../C-8b-h100-cold/scheduler.log | 2 + .../C-8b-h100-cold/timeline.json | 275 + .../C-8b-h100-cold/timeline.md | 111 + .../trainer-mountstats-sample1.txt | 14 + .../trainer-mountstats-sample2.txt | 2 + .../C-8b-h100-cold/watch-started.txt | 1 + .../C-8b-h100-cold/worker-image.txt | 1 + .../C-8b-h100-cold/workloads.json | 355 + .../D-8b-h100-cached-interrupted/NOTE.txt | 1 + .../api-server.log | 38 + .../D-8b-h100-cached-interrupted/client.jsonl | 4 + .../D-8b-h100-cached-interrupted/client.log | 6 + .../collect-started.txt | 1 + .../D-8b-h100-cached-interrupted/events.json | 401 + .../node-images-after.json | 50 + .../node-images-before.json | 50 + .../node-labels-during.txt | 8 + .../D-8b-h100-cached-interrupted/pods.json | 5980 +++++++++++++++ .../redis-ready.log | 1001 +++ .../D-8b-h100-cached-interrupted/run.log | 36 + .../scheduler.log | 2 + .../timeline.json | 26 + .../D-8b-h100-cached-interrupted/timeline.md | 54 + .../watch-started.txt | 1 + .../worker-image.txt | 1 + .../workloads.json | 8 + .../D-8b-h100-cached.run.log | 35 + .../D-8b-h100-cached/api-server.log | 36 + .../D-8b-h100-cached/client.jsonl | 11 + .../D-8b-h100-cached/client.log | 12 + .../D-8b-h100-cached/collect-started.txt | 1 + .../D-8b-h100-cached/events.json | 661 ++ .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 223 + .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 166 + .../D-8b-h100-cached/monitoring.json | 194 + .../D-8b-h100-cached/monitoring.md | 7 + .../D-8b-h100-cached/node-images-after.json | 50 + .../D-8b-h100-cached/node-images-before.json | 50 + .../D-8b-h100-cached/node-labels-during.txt | 8 + .../D-8b-h100-cached/pods.json | 6782 ++++++++++++++++ .../D-8b-h100-cached/redis-ready.log | 147 + .../D-8b-h100-cached/scheduler.log | 2 + .../D-8b-h100-cached/timeline.json | 267 + .../D-8b-h100-cached/timeline.md | 109 + .../D-8b-h100-cached/watch-started.txt | 1 + .../D-8b-h100-cached/worker-image.txt | 1 + .../D-8b-h100-cached/workloads.json | 355 + runs/2026-10-15_cold_start/REPORT.md | 239 + .../after-A1-0.5b-l4-cold.run.log | 35 + .../after-A1-0.5b-l4-cold/api-server.log | 75 + .../after-A1-0.5b-l4-cold/client.jsonl | 11 + .../after-A1-0.5b-l4-cold/client.log | 12 + .../after-A1-0.5b-l4-cold/collect-started.txt | 1 + .../after-A1-0.5b-l4-cold/events.json | 1311 ++++ ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 + ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 34 + .../node-images-after.json | 49 + .../node-images-before.json | 51 + .../node-labels-during.txt | 8 + .../after-A1-0.5b-l4-cold/pods.json | 6782 ++++++++++++++++ .../after-A1-0.5b-l4-cold/redis-ready.log | 309 + .../after-A1-0.5b-l4-cold/scheduler.log | 2 + .../after-A1-0.5b-l4-cold/timeline.json | 275 + .../after-A1-0.5b-l4-cold/timeline.md | 111 + .../after-A1-0.5b-l4-cold/watch-started.txt | 1 + .../after-A1-0.5b-l4-cold/worker-image.txt | 1 + .../after-A1-0.5b-l4-cold/workloads.json | 355 + .../after-B1-0.5b-l4-cached.run.log | 35 + .../after-B1-0.5b-l4-cached/api-server.log | 33 + .../after-B1-0.5b-l4-cached/client.jsonl | 11 + .../after-B1-0.5b-l4-cached/client.log | 12 + .../collect-started.txt | 1 + .../after-B1-0.5b-l4-cached/events.json | 1699 +++++ ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 119 + ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 34 + .../node-images-after.json | 49 + .../node-images-before.json | 49 + .../node-labels-during.txt | 8 + .../after-B1-0.5b-l4-cached/pods.json | 6786 +++++++++++++++++ .../after-B1-0.5b-l4-cached/redis-ready.log | 175 + .../after-B1-0.5b-l4-cached/scheduler.log | 2 + .../after-B1-0.5b-l4-cached/timeline.json | 275 + .../after-B1-0.5b-l4-cached/timeline.md | 111 + .../after-B1-0.5b-l4-cached/watch-started.txt | 1 + .../after-B1-0.5b-l4-cached/worker-image.txt | 1 + .../after-B1-0.5b-l4-cached/workloads.json | 355 + .../after-B2-0.5b-l4-cached.run.log | 35 + .../after-B2-0.5b-l4-cached/api-server.log | 30 + .../after-B2-0.5b-l4-cached/client.jsonl | 11 + .../after-B2-0.5b-l4-cached/client.log | 12 + .../collect-started.txt | 1 + .../after-B2-0.5b-l4-cached/events.json | 2025 +++++ ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 + ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 34 + .../node-images-after.json | 49 + .../node-images-before.json | 49 + .../node-labels-during.txt | 8 + .../after-B2-0.5b-l4-cached/pods.json | 6782 ++++++++++++++++ .../after-B2-0.5b-l4-cached/redis-ready.log | 153 + .../after-B2-0.5b-l4-cached/scheduler.log | 2 + .../after-B2-0.5b-l4-cached/timeline.json | 267 + .../after-B2-0.5b-l4-cached/timeline.md | 109 + .../after-B2-0.5b-l4-cached/watch-started.txt | 1 + .../after-B2-0.5b-l4-cached/worker-image.txt | 1 + .../after-B2-0.5b-l4-cached/workloads.json | 355 + .../after-C1-8b-h100-cold.run.log | 35 + .../after-C1-8b-h100-cold/api-server.log | 56 + .../after-C1-8b-h100-cold/client.jsonl | 11 + .../after-C1-8b-h100-cold/client.log | 12 + .../after-C1-8b-h100-cold/collect-started.txt | 1 + .../after-C1-8b-h100-cold/events.json | 2417 ++++++ .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 223 + .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 50 + .../after-C1-8b-h100-cold/monitoring.json | 320 + .../after-C1-8b-h100-cold/monitoring.md | 11 + .../node-images-after.json | 51 + .../node-images-before.json | 49 + .../node-labels-during.txt | 8 + .../after-C1-8b-h100-cold/pods.json | 6782 ++++++++++++++++ .../after-C1-8b-h100-cold/redis-ready.log | 575 ++ .../after-C1-8b-h100-cold/scheduler.log | 2 + .../after-C1-8b-h100-cold/timeline.json | 275 + .../after-C1-8b-h100-cold/timeline.md | 111 + .../after-C1-8b-h100-cold/watch-started.txt | 1 + .../after-C1-8b-h100-cold/worker-image.txt | 1 + .../after-C1-8b-h100-cold/workloads.json | 355 + .../after-D1-8b-h100-cached.run.log | 35 + .../after-D1-8b-h100-cached/api-server.log | 32 + .../after-D1-8b-h100-cached/client.jsonl | 11 + .../after-D1-8b-h100-cached/client.log | 12 + .../collect-started.txt | 1 + .../after-D1-8b-h100-cached/events.json | 2743 +++++++ .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 222 + .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 50 + .../node-images-after.json | 51 + .../node-images-before.json | 51 + .../node-labels-during.txt | 8 + .../after-D1-8b-h100-cached/pods.json | 6782 ++++++++++++++++ .../after-D1-8b-h100-cached/redis-ready.log | 342 + .../after-D1-8b-h100-cached/scheduler.log | 2 + .../after-D1-8b-h100-cached/timeline.json | 267 + .../after-D1-8b-h100-cached/timeline.md | 109 + .../after-D1-8b-h100-cached/watch-started.txt | 1 + .../after-D1-8b-h100-cached/worker-image.txt | 1 + .../after-D1-8b-h100-cached/workloads.json | 355 + .../after-D2-8b-h100-cached.run.log | 35 + .../after-D2-8b-h100-cached/api-server.log | 32 + .../after-D2-8b-h100-cached/client.jsonl | 11 + .../after-D2-8b-h100-cached/client.log | 12 + .../collect-started.txt | 1 + .../after-D2-8b-h100-cached/events.json | 3069 ++++++++ .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 222 + .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 50 + .../node-images-after.json | 51 + .../node-images-before.json | 51 + .../node-labels-during.txt | 8 + .../after-D2-8b-h100-cached/pods.json | 6782 ++++++++++++++++ .../after-D2-8b-h100-cached/redis-ready.log | 334 + .../after-D2-8b-h100-cached/scheduler.log | 2 + .../after-D2-8b-h100-cached/timeline.json | 267 + .../after-D2-8b-h100-cached/timeline.md | 109 + .../after-D2-8b-h100-cached/watch-started.txt | 1 + .../after-D2-8b-h100-cached/worker-image.txt | 1 + .../after-D2-8b-h100-cached/workloads.json | 355 + .../api-server.log | 26 + .../client.jsonl | 7 + .../client.log | 8 + .../collect-started.txt | 1 + .../events.json | 3265 ++++++++ ...-orw-lora-google-gemma-4-e2b-0-trainer.txt | 32 + .../monitoring.json | 158 + .../monitoring.md | 5 + .../node-images-after.json | 51 + .../node-images-before.json | 51 + .../pods.json | 6378 ++++++++++++++++ .../redis-ready.log | 258 + .../scheduler.log | 1 + .../timeline.json | 130 + .../timeline.md | 78 + .../trainer-mountstats-final.txt | 3 + .../watch-started.txt | 1 + .../worker-image.txt | 1 + .../workloads.json | 179 + runs/2026-10-15_cold_start/after-progress.txt | 7 + runs/2026-10-15_cold_start/after.log | 0 runs/2026-10-15_cold_start/after.sh | 15 + .../fio-h100-1g3a-coldfile/file.txt | 1 + .../fio-randread-4k-direct.json | 260 + .../fio-seqread-1M-4jobs-direct.json | 261 + .../fio-seqread-1M-buffered.json | 260 + .../fio-seqread-1M-direct.json | 260 + .../fio-seqread-256k-qd1-direct.json | 260 + .../fio-h100-1g3a-coldfile/monitoring.md | 11 + .../mountstats-after.txt | 31 + .../mountstats-before.txt | 32 + .../fio-h100-1g3a-coldfile/summary.md | 11 + .../fio-l4-lvw8/file.txt | 1 + .../fio-l4-lvw8/fio-randread-4k-direct.json | 260 + .../fio-seqread-1M-4jobs-direct.json | 261 + .../fio-l4-lvw8/fio-seqread-1M-buffered.json | 260 + .../fio-l4-lvw8/fio-seqread-1M-direct.json | 260 + .../fio-seqread-256k-qd1-direct.json | 260 + .../fio-l4-lvw8/monitoring.json | 302 + .../fio-l4-lvw8/monitoring.md | 11 + .../fio-l4-lvw8/mountstats-after.txt | 31 + .../fio-l4-lvw8/mountstats-before.txt | 32 + .../fio-l4-lvw8/summary.md | 11 + runs/2026-10-15_cold_start/fix-image-tag.txt | 1 + .../node-labels-before.json | 42 + runs/2026-10-15_cold_start/summary.md | 26 + scripts/cold_start_fio.sh | 6 +- scripts/cold_start_monitoring.py | 73 + scripts/cold_start_resident.py | 52 + scripts/cold_start_summary.py | 74 + scripts/cold_start_timeline.py | 65 +- 272 files changed, 123801 insertions(+), 15 deletions(-) create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/monitoring.json create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/monitoring.md create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.md create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/monitoring.json create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/monitoring.md create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/pods.json create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/scheduler.log create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/timeline.json create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/timeline.md create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/workloads.json create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold.run.log create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/api-server.log create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/client.jsonl create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/client.log create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/events.json create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/monitoring.json create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/monitoring.md create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.md create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.md create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached.run.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/client.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/events.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/monitoring.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/monitoring.md create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.md create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json create mode 100644 runs/2026-10-15_cold_start/REPORT.md create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.md create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.md create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.md create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/monitoring.json create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/monitoring.md create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.md create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.md create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/node-labels-during.txt create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.md create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/monitoring.json create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/monitoring.md create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/node-images-after.json create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/node-images-before.json create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/pods.json create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/redis-ready.log create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/scheduler.log create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/timeline.json create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/timeline.md create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/trainer-mountstats-final.txt create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/watch-started.txt create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/worker-image.txt create mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/workloads.json create mode 100644 runs/2026-10-15_cold_start/after-progress.txt create mode 100644 runs/2026-10-15_cold_start/after.log create mode 100755 runs/2026-10-15_cold_start/after.sh create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/monitoring.md create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt create mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/summary.md create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.md create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt create mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/summary.md create mode 100644 runs/2026-10-15_cold_start/fix-image-tag.txt create mode 100644 runs/2026-10-15_cold_start/node-labels-before.json create mode 100644 runs/2026-10-15_cold_start/summary.md create mode 100644 scripts/cold_start_monitoring.py create mode 100644 scripts/cold_start_resident.py create mode 100644 scripts/cold_start_summary.py diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log new file mode 100644 index 00000000..f232d67d --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log @@ -0,0 +1,34 @@ +watching; recorders [95336, 95337, 95339, 95341, 95343, 95338, 95340, 95342] write into runs/2026-10-15_cold_start/A-0.5b-l4-cold +collected 2 worker log(s) into runs/2026-10-15_cold_start/A-0.5b-l4-cold +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.1 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 1.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | -300.0 | +| container create | trainer.event_Pulled | trainer.container_started | 473.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.4 | +| weights load | trainer.load_start | trainer.load_done | 4.3 | +| ready key | trainer.load_done | trainer.ready_key | -0.7 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 186.7 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.4 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.6 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 1.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | -470.0 | +| container create | sampler.event_Pulled | sampler.container_started | 642.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 11.9 | +| imports and engine setup until weights read | sampler.process_first_log | sampler.weights_loaded | 32.7 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.5 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.6 | +| ready key | sampler.engine_init_done | sampler.ready_key | 1.2 | +| client: first sample blocked | client.sample.start | client.sample.done | 277.0 | + +DONE=0 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log new file mode 100644 index 00000000..d10c262a --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log @@ -0,0 +1,71 @@ +2026-10-15T03:04:13.970+00:00 2026-10-15T03:04:13.965424776Z INFO: 10.32.3.1:49232 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:04:16.136+00:00 2026-10-15T03:04:16.130670078Z INFO: 127.0.0.1:55612 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T03:04:16.365+00:00 2026-10-15T03:04:16.360351230Z INFO: 127.0.0.1:55616 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T03:04:16.427+00:00 2026-10-15T03:04:16.423488415Z INFO: 127.0.0.1:55616 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T03:04:16.648+00:00 2026-10-15T03:04:16.643839900Z [API_SERVER] enqueued op=create_model request_id=ccc385bd-2c80-4be6-9ab7-0625542435ba model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T03:04:16.650+00:00 2026-10-15T03:04:16.644342790Z INFO: 127.0.0.1:55632 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T03:04:23.971+00:00 2026-10-15T03:04:23.964124117Z INFO: 10.32.3.1:36036 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:04:26.615+00:00 2026-10-15T03:04:26.609759344Z INFO: 127.0.0.1:48514 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:04:34.028+00:00 2026-10-15T03:04:33.964594425Z INFO: 10.32.3.1:33560 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:04:43.971+00:00 2026-10-15T03:04:43.964844670Z INFO: 10.32.3.1:40012 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:04:53.971+00:00 2026-10-15T03:04:53.964767187Z INFO: 10.32.3.1:40016 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:05:03.980+00:00 2026-10-15T03:05:03.965100240Z INFO: 10.32.3.1:34000 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:05:06.801+00:00 2026-10-15T03:05:06.795756734Z INFO: 127.0.0.1:42920 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:05:13.972+00:00 2026-10-15T03:05:13.964966273Z INFO: 10.32.3.1:55820 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:05:23.970+00:00 2026-10-15T03:05:23.965043231Z INFO: 10.32.3.1:48518 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:05:33.972+00:00 2026-10-15T03:05:33.965111192Z INFO: 10.32.3.1:56412 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:05:43.970+00:00 2026-10-15T03:05:43.964441881Z INFO: 10.32.3.1:40700 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:05:53.970+00:00 2026-10-15T03:05:53.964733835Z INFO: 10.32.3.1:59024 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:05:56.984+00:00 2026-10-15T03:05:56.978355679Z INFO: 127.0.0.1:38040 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:06:04.141+00:00 2026-10-15T03:06:03.965149122Z INFO: 10.32.3.1:54646 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:06:13.983+00:00 2026-10-15T03:06:13.964712249Z INFO: 10.32.3.1:48236 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:06:23.971+00:00 2026-10-15T03:06:23.964370498Z INFO: 10.32.3.1:44162 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:06:33.971+00:00 2026-10-15T03:06:33.965253254Z INFO: 10.32.3.1:49780 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:06:37.171+00:00 2026-10-15T03:06:37.165060181Z INFO: 127.0.0.1:34956 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:06:43.970+00:00 2026-10-15T03:06:43.964900145Z INFO: 10.32.3.1:44846 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:06:53.971+00:00 2026-10-15T03:06:53.964366759Z INFO: 10.32.3.1:58212 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:07:03.971+00:00 2026-10-15T03:07:03.964671419Z INFO: 10.32.3.1:59690 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:07:13.971+00:00 2026-10-15T03:07:13.964489989Z INFO: 10.32.3.1:46040 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:07:22.817+00:00 2026-10-15T03:07:22.810967004Z INFO: 127.0.0.1:59842 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T03:07:24.013+00:00 2026-10-15T03:07:23.965218186Z INFO: 10.32.3.1:47190 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:07:25.486+00:00 2026-10-15T03:07:25.480772722Z [API_SERVER] enqueued op=forward_backward request_id=c9b1b9ba-be87-47ed-8531-402cb2fabb4c model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T03:07:25.487+00:00 2026-10-15T03:07:25.481519493Z INFO: 127.0.0.1:59842 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T03:07:28.911+00:00 2026-10-15T03:07:28.906017913Z [API_SERVER] enqueued op=optim_step request_id=4eed00a3-56a3-46ac-9191-13fd5f7bcf0d model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T03:07:28.911+00:00 2026-10-15T03:07:28.906487435Z INFO: 127.0.0.1:59842 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T03:07:29.701+00:00 2026-10-15T03:07:29.697228415Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=0e4cfa12-b1fd-41e9-8346-b31a02c5d55f model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T03:07:29.703+00:00 2026-10-15T03:07:29.697821936Z INFO: 127.0.0.1:59842 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T03:07:30.370+00:00 2026-10-15T03:07:30.365760850Z INFO: 127.0.0.1:59860 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T03:07:33.971+00:00 2026-10-15T03:07:33.966052502Z INFO: 10.32.3.1:47144 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:07:43.972+00:00 2026-10-15T03:07:43.965551746Z INFO: 10.32.3.1:39724 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:07:53.974+00:00 2026-10-15T03:07:53.966118583Z INFO: 10.32.3.1:52036 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:08:04.003+00:00 2026-10-15T03:08:03.965792751Z INFO: 10.32.3.1:53772 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:08:13.972+00:00 2026-10-15T03:08:13.965952960Z INFO: 10.32.3.1:55912 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:08:17.378+00:00 2026-10-15T03:08:17.363707043Z INFO: 127.0.0.1:40350 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:08:23.972+00:00 2026-10-15T03:08:23.965275434Z INFO: 10.32.3.1:37596 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:08:34.056+00:00 2026-10-15T03:08:33.964948070Z INFO: 10.32.3.1:55858 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:08:43.973+00:00 2026-10-15T03:08:43.966182718Z INFO: 10.32.3.1:33098 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:08:53.973+00:00 2026-10-15T03:08:53.965614609Z INFO: 10.32.3.1:38942 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:09:03.972+00:00 2026-10-15T03:09:03.964678901Z INFO: 10.32.3.1:45794 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:09:07.564+00:00 2026-10-15T03:09:07.556603346Z INFO: 127.0.0.1:47276 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:09:13.976+00:00 2026-10-15T03:09:13.967273256Z INFO: 10.32.3.1:54252 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:09:16.493+00:00 2026-10-15T03:09:16.484867631Z INFO: 127.0.0.1:57588 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T03:09:23.972+00:00 2026-10-15T03:09:23.964796053Z INFO: 10.32.3.1:60326 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:09:33.973+00:00 2026-10-15T03:09:33.965636084Z INFO: 10.32.3.1:58520 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:09:43.973+00:00 2026-10-15T03:09:43.965716052Z INFO: 10.32.3.1:46004 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:09:53.976+00:00 2026-10-15T03:09:53.966124758Z INFO: 10.32.3.1:58072 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:09:57.751+00:00 2026-10-15T03:09:57.744852851Z INFO: 127.0.0.1:60840 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:10:03.975+00:00 2026-10-15T03:10:03.966670840Z INFO: 10.32.3.1:58060 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:10:13.975+00:00 2026-10-15T03:10:13.966758337Z INFO: 10.32.3.1:57718 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:10:23.974+00:00 2026-10-15T03:10:23.964882780Z INFO: 10.32.3.1:33186 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:10:33.975+00:00 2026-10-15T03:10:33.966514608Z INFO: 10.32.3.1:45636 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:10:37.927+00:00 2026-10-15T03:10:37.918752812Z INFO: 127.0.0.1:33472 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:10:43.984+00:00 2026-10-15T03:10:43.965936582Z INFO: 10.32.3.1:50548 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:10:53.974+00:00 2026-10-15T03:10:53.965049215Z INFO: 10.32.3.1:56646 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:11:03.975+00:00 2026-10-15T03:11:03.965725149Z INFO: 10.32.3.1:46908 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:11:13.971+00:00 2026-10-15T03:11:13.964776008Z INFO: 10.32.3.1:59014 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:11:23.973+00:00 2026-10-15T03:11:23.964881664Z INFO: 10.32.3.1:37578 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:11:33.972+00:00 2026-10-15T03:11:33.964840171Z INFO: 10.32.3.1:57742 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:11:38.106+00:00 2026-10-15T03:11:38.100543048Z INFO: 127.0.0.1:49588 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:11:43.973+00:00 2026-10-15T03:11:43.965228717Z INFO: 10.32.3.1:60810 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:11:53.972+00:00 2026-10-15T03:11:53.964561540Z INFO: 10.32.3.1:56102 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:12:03.973+00:00 2026-10-15T03:12:03.965618560Z INFO: 10.32.3.1:33308 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl new file mode 100644 index 00000000..2ab7d41f --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T03:04:15.891+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T03:04:15.892+00:00", "t": 0.0, "phase": "create_model.start"} +{"ts": "2026-10-15T03:07:22.617+00:00", "t": 186.725, "phase": "create_model.done", "model_id": "ccc385bd-2c80-4be6-9ab7-0625542435ba"} +{"ts": "2026-10-15T03:07:25.417+00:00", "t": 189.525, "phase": "forward_backward.start"} +{"ts": "2026-10-15T03:07:28.837+00:00", "t": 192.945, "phase": "forward_backward.done"} +{"ts": "2026-10-15T03:07:29.600+00:00", "t": 193.709, "phase": "optim_step.done"} +{"ts": "2026-10-15T03:07:29.600+00:00", "t": 193.709, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T03:07:30.173+00:00", "t": 194.282, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T03:07:30.174+00:00", "t": 194.282, "phase": "sample.start"} +{"ts": "2026-10-15T03:12:07.151+00:00", "t": 471.26, "phase": "sample.done", "text": " 4"} +{"ts": "2026-10-15T03:12:07.154+00:00", "t": 471.262, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log new file mode 100644 index 00000000..0f6171ae --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.000s create_model.start +[cold-start] 186.725s create_model.done {'model_id': 'ccc385bd-2c80-4be6-9ab7-0625542435ba'} +[cold-start] 189.525s forward_backward.start +[cold-start] 192.945s forward_backward.done +[cold-start] 193.709s optim_step.done +[cold-start] 193.709s save_weights_and_get_sampling_client.start +[cold-start] 194.282s save_weights_and_get_sampling_client.done +[cold-start] 194.282s sample.start +[cold-start] 471.260s sample.done {'text': ' 4'} +[cold-start] 471.262s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt new file mode 100644 index 00000000..45604b3b --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T03:12:07.681+00:00 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json new file mode 100644 index 00000000..74c9ca47 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json @@ -0,0 +1,661 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:36Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176697983001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:36Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:36Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94ae008553cd", + "namespace": "openrl-system", + "resourceVersion": "1792033176815007016", + "uid": "363bd5d3-147c-4b0d-8500-d3a3b6232789" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:40Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176788543001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:40Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:40Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed139d233", + "namespace": "openrl-system", + "resourceVersion": "1792033180304687014", + "uid": "2463b9b8-32de-4c33-af86-ca945ab359e3" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:40Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176788543001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:40Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:40Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed2f06544", + "namespace": "openrl-system", + "resourceVersion": "1792033180345279015", + "uid": "4766979d-5fb4-4934-96f7-b8e5c1833efa" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:40Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176788543001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:40Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:40Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aedddd6086", + "namespace": "openrl-system", + "resourceVersion": "1792033180517807014", + "uid": "e1c7e13f-7b29-4ed9-afdb-cc1a062a32d2" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:03:24Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176788543001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:03:24Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:03:24Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94e2ff112e95", + "namespace": "openrl-system", + "resourceVersion": "1792033404430399002", + "uid": "2f914e49-5325-4325-b2af-29bca754d7d2" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:07:30Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033649966431001", + "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:07:30Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T03:07:30Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de951c3308c78d", + "namespace": "openrl-system", + "resourceVersion": "1792033650098607013", + "uid": "703ebdf3-49c5-40b9-abd8-9ae5be2441e0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:07:33Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033650081775001", + "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:07:33Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", + "metadata": { + "creationTimestamp": "2026-10-15T03:07:33Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de951d16e18ede", + "namespace": "openrl-system", + "resourceVersion": "1792033653954879017", + "uid": "b6907b80-ca84-45a3-bf35-1cc63a9f9e88" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:10:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033650081775001", + "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:10:22Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.566s (2m48.566s including waiting). Image size: 7123551739 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T03:10:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de954456385799", + "namespace": "openrl-system", + "resourceVersion": "1792033822509983001", + "uid": "14bc5358-d229-48a6-aea1-586d2b488ba2" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:10:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033650081775001", + "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:10:22Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T03:10:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de95445d54c744", + "namespace": "openrl-system", + "resourceVersion": "1792033822614143018", + "uid": "c54769e6-3c1e-4740-9a25-03c3b3f2f2c8" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:10:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033650081775001", + "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:10:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T03:10:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de954472a82f0e", + "namespace": "openrl-system", + "resourceVersion": "1792033822991583015", + "uid": "3e74963a-d669-4b13-8870-db29be3feee1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153094671012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a883fadef0", + "namespace": "openrl-system", + "resourceVersion": "1792033153265231011", + "uid": "62c5cae6-2deb-43a0-a097-73157e760539" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:17Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153224047012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:17Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:17Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a9671f24f0", + "namespace": "openrl-system", + "resourceVersion": "1792033157084143008", + "uid": "9472795b-31b9-478f-a545-6146795d5503" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:17Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153224047012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:17Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:17Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a968b7d635", + "namespace": "openrl-system", + "resourceVersion": "1792033157103615024", + "uid": "11d1cf36-8464-42c3-94e2-1843723223c8" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:17Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153224047012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:17Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:17Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a973285d70", + "namespace": "openrl-system", + "resourceVersion": "1792033157264271007", + "uid": "8fa654a4-608f-4e8c-804f-9d1d5c91650a" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:03:23Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153224047012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:03:23Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:03:23Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94e2d9f9f25a", + "namespace": "openrl-system", + "resourceVersion": "1792033403812895007", + "uid": "acb91806-a2a8-401c-af8a-60b6035618d4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:04:17Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033457005855012", + "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:04:17Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T03:04:17Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94ef537aea10", + "namespace": "openrl-system", + "resourceVersion": "1792033457378303014", + "uid": "c2b1e3e0-7483-4202-a802-114b81a3f82b" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:04:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033457353199012", + "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:04:21Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", + "metadata": { + "creationTimestamp": "2026-10-15T03:04:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94f037d8b52d", + "namespace": "openrl-system", + "resourceVersion": "1792033461208399017", + "uid": "b6d87d48-8207-4dc8-95cb-72d018a65a89" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:07:09Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033457353199012", + "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:07:09Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.59s (2m48.59s including waiting). Image size: 7123551739 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T03:07:09Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de951778a75e12", + "namespace": "openrl-system", + "resourceVersion": "1792033629797471021", + "uid": "9f72c5d6-3d2e-40cc-97a1-d829da2bdcf6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:07:09Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033457353199012", + "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:07:09Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T03:07:09Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de95177ce47bff", + "namespace": "openrl-system", + "resourceVersion": "1792033629877183016", + "uid": "cff96884-931d-4efe-9b36-7e38e39d1ab6" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:07:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033457353199012", + "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:07:10Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T03:07:10Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de95178de8c928", + "namespace": "openrl-system", + "resourceVersion": "1792033630159423023", + "uid": "99e1e2c0-7347-45f8-ae1d-2c56266651d1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt new file mode 100644 index 00000000..9affdaaf --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt @@ -0,0 +1,120 @@ +2026-10-15T03:10:33.926555905Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T03:10:46.387987219Z INFO 10-15 03:10:46 [model.py:692] Resolved architecture: Qwen2ForCausalLM +2026-10-15T03:10:46.388270451Z INFO 10-15 03:10:46 [model.py:2030] Using max model len 8192 +2026-10-15T03:10:46.390489193Z INFO 10-15 03:10:46 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. +2026-10-15T03:10:46.761209045Z INFO 10-15 03:10:46 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T03:10:50.185230245Z WARNING 10-15 03:10:50 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T03:11:01.668050858Z (EngineCore pid=161) INFO 10-15 03:11:01 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T03:11:02.480729209Z (EngineCore pid=161) INFO 10-15 03:11:02 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_af67dc74611246bb99d63df9f6327803 backend=nccl +2026-10-15T03:11:02.487863050Z (EngineCore pid=161) INFO 10-15 03:11:02 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T03:11:02.487892790Z (EngineCore pid=161) INFO 10-15 03:11:02 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T03:11:03.371362030Z (EngineCore pid=161) INFO 10-15 03:11:03 [model_runner.py:396] Loading model from scratch... +2026-10-15T03:11:04.235459935Z (EngineCore pid=161) INFO 10-15 03:11:04 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T03:11:04.235903689Z (EngineCore pid=161) INFO 10-15 03:11:04 [flash_attn.py:1116] Using FlashAttention version 2 +2026-10-15T03:11:04.354712405Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T03:11:04.355179834Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T03:11:04.731296224Z (EngineCore pid=161) INFO 10-15 03:11:04 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.62 GiB. +2026-10-15T03:11:04.731338778Z (EngineCore pid=161) INFO 10-15 03:11:04 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T03:11:04.732630306Z (EngineCore pid=161) +Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T03:07:17.275476076Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T03:07:17.307183476Z +2026-10-15T03:07:17.410020160Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... +2026-10-15T03:07:17.543492303Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T03:07:21.568855096Z +Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 24h v1.35.8-gke.1380001 false +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 24h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json new file mode 100644 index 00000000..71e4ba3c --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-api-server-645ff9777d-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "645ff9777d", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + } + ], + "resourceVersion": "1791941173996783021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-79xdl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:02Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:26:02Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.10", + "podIPs": [ + { + "ip": "10.32.3.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T03:07:29Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "uid": "54443ec4-9700-416e-b5d7-846a4a007c00" + } + ], + "resourceVersion": "1792033823293583001", + "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "d96cccda-acfd-4fa4-95c5-72c958a83f6b" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zfr8c", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-zfr8c", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:10:23Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:07:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:10:23Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:10:23Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:07:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "25Gi" + }, + "containerID": "containerd://2183700bb5c7f07cd82180b35a53253d88b404e2508623e204e56ec7fc76674b", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T03:10:22Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zfr8c", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.0.19", + "podIPs": [ + { + "ip": "10.32.0.19" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T03:07:30Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T03:04:16Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "uid": "47b54db3-9dc6-4dfd-a972-55bee1fc0dad" + } + ], + "resourceVersion": "1792033631097647012", + "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "33cc03f0-42b9-4860-8d5e-f14ca74f3511" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2b24h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2b24h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:07:11Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:04:17Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:07:11Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:07:11Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:04:17Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "21Gi" + }, + "containerID": "containerd://1e1957d4c7ed76e3bcb86d4334323d28c34b994664a0e8e29b4f551d8f8f5b99", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T03:07:10Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2b24h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.13.24", + "podIPs": [ + { + "ip": "10.32.13.24" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T03:04:17Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log new file mode 100644 index 00000000..a3b2f618 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log @@ -0,0 +1,299 @@ +2026-10-15T03:07:21.981+00:00 2026-10-15T03:07:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:22.984+00:00 2026-10-15T03:07:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:24.013+00:00 2026-10-15T03:07:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:24.993+00:00 2026-10-15T03:07:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:25.999+00:00 2026-10-15T03:07:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:27.005+00:00 2026-10-15T03:07:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:28.011+00:00 2026-10-15T03:07:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:29.016+00:00 2026-10-15T03:07:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:30.021+00:00 2026-10-15T03:07:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:31.026+00:00 2026-10-15T03:07:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:32.031+00:00 2026-10-15T03:07:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:33.036+00:00 2026-10-15T03:07:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:34.041+00:00 2026-10-15T03:07:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:35.046+00:00 2026-10-15T03:07:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:36.051+00:00 2026-10-15T03:07:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:37.056+00:00 2026-10-15T03:07:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:38.062+00:00 2026-10-15T03:07:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:39.068+00:00 2026-10-15T03:07:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:40.072+00:00 2026-10-15T03:07:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:41.077+00:00 2026-10-15T03:07:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:42.083+00:00 2026-10-15T03:07:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:43.088+00:00 2026-10-15T03:07:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:44.098+00:00 2026-10-15T03:07:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:45.098+00:00 2026-10-15T03:07:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:46.105+00:00 2026-10-15T03:07:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:47.109+00:00 2026-10-15T03:07:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:48.116+00:00 2026-10-15T03:07:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:49.118+00:00 2026-10-15T03:07:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:50.124+00:00 2026-10-15T03:07:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:51.129+00:00 2026-10-15T03:07:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:52.139+00:00 2026-10-15T03:07:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:53.140+00:00 2026-10-15T03:07:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:54.145+00:00 2026-10-15T03:07:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:55.150+00:00 2026-10-15T03:07:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:56.154+00:00 2026-10-15T03:07:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:57.163+00:00 2026-10-15T03:07:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:58.164+00:00 2026-10-15T03:07:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:07:59.169+00:00 2026-10-15T03:07:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:00.174+00:00 2026-10-15T03:08:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:01.180+00:00 2026-10-15T03:08:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:02.184+00:00 2026-10-15T03:08:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:03.190+00:00 2026-10-15T03:08:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:04.194+00:00 2026-10-15T03:08:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:05.213+00:00 2026-10-15T03:08:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:06.413+00:00 2026-10-15T03:08:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:07.208+00:00 2026-10-15T03:08:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:08.214+00:00 2026-10-15T03:08:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:09.219+00:00 2026-10-15T03:08:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:10.235+00:00 2026-10-15T03:08:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:11.229+00:00 2026-10-15T03:08:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:12.234+00:00 2026-10-15T03:08:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:13.248+00:00 2026-10-15T03:08:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:14.245+00:00 2026-10-15T03:08:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:15.252+00:00 2026-10-15T03:08:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:16.257+00:00 2026-10-15T03:08:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:17.262+00:00 2026-10-15T03:08:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:18.269+00:00 2026-10-15T03:08:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:19.272+00:00 2026-10-15T03:08:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:20.284+00:00 2026-10-15T03:08:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:21.282+00:00 2026-10-15T03:08:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:22.289+00:00 2026-10-15T03:08:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:23.292+00:00 2026-10-15T03:08:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:24.298+00:00 2026-10-15T03:08:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:25.455+00:00 2026-10-15T03:08:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:26.309+00:00 2026-10-15T03:08:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:27.313+00:00 2026-10-15T03:08:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:28.320+00:00 2026-10-15T03:08:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:29.330+00:00 2026-10-15T03:08:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:30.328+00:00 2026-10-15T03:08:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:31.333+00:00 2026-10-15T03:08:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:32.339+00:00 2026-10-15T03:08:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:33.342+00:00 2026-10-15T03:08:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:34.352+00:00 2026-10-15T03:08:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:35.362+00:00 2026-10-15T03:08:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:36.358+00:00 2026-10-15T03:08:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:37.362+00:00 2026-10-15T03:08:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:38.367+00:00 2026-10-15T03:08:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:39.374+00:00 2026-10-15T03:08:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:40.382+00:00 2026-10-15T03:08:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:41.382+00:00 2026-10-15T03:08:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:42.388+00:00 2026-10-15T03:08:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:43.392+00:00 2026-10-15T03:08:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:44.398+00:00 2026-10-15T03:08:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:45.403+00:00 2026-10-15T03:08:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:46.408+00:00 2026-10-15T03:08:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:47.417+00:00 2026-10-15T03:08:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:48.419+00:00 2026-10-15T03:08:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:49.452+00:00 2026-10-15T03:08:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:50.428+00:00 2026-10-15T03:08:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:51.436+00:00 2026-10-15T03:08:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:52.440+00:00 2026-10-15T03:08:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:53.445+00:00 2026-10-15T03:08:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:54.451+00:00 2026-10-15T03:08:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:55.456+00:00 2026-10-15T03:08:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:56.465+00:00 2026-10-15T03:08:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:57.465+00:00 2026-10-15T03:08:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:58.470+00:00 2026-10-15T03:08:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:08:59.476+00:00 2026-10-15T03:08:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:00.480+00:00 2026-10-15T03:09:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:01.484+00:00 2026-10-15T03:09:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:02.490+00:00 2026-10-15T03:09:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:03.495+00:00 2026-10-15T03:09:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:04.501+00:00 2026-10-15T03:09:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:05.511+00:00 2026-10-15T03:09:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:06.510+00:00 2026-10-15T03:09:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:07.516+00:00 2026-10-15T03:09:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:08.653+00:00 2026-10-15T03:09:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:09.524+00:00 2026-10-15T03:09:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:10.667+00:00 2026-10-15T03:09:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:11.537+00:00 2026-10-15T03:09:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:12.540+00:00 2026-10-15T03:09:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:13.546+00:00 2026-10-15T03:09:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:14.556+00:00 2026-10-15T03:09:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:15.556+00:00 2026-10-15T03:09:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:16.562+00:00 2026-10-15T03:09:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:17.565+00:00 2026-10-15T03:09:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:18.704+00:00 2026-10-15T03:09:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:19.576+00:00 2026-10-15T03:09:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:20.581+00:00 2026-10-15T03:09:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:21.589+00:00 2026-10-15T03:09:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:22.637+00:00 2026-10-15T03:09:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:23.599+00:00 2026-10-15T03:09:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:24.603+00:00 2026-10-15T03:09:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:25.608+00:00 2026-10-15T03:09:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:26.613+00:00 2026-10-15T03:09:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:27.629+00:00 2026-10-15T03:09:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:28.628+00:00 2026-10-15T03:09:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:29.628+00:00 2026-10-15T03:09:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:30.633+00:00 2026-10-15T03:09:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:31.638+00:00 2026-10-15T03:09:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:32.642+00:00 2026-10-15T03:09:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:33.648+00:00 2026-10-15T03:09:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:34.653+00:00 2026-10-15T03:09:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:35.659+00:00 2026-10-15T03:09:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:36.728+00:00 2026-10-15T03:09:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:37.669+00:00 2026-10-15T03:09:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:38.674+00:00 2026-10-15T03:09:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:39.679+00:00 2026-10-15T03:09:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:40.686+00:00 2026-10-15T03:09:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:41.850+00:00 2026-10-15T03:09:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:42.695+00:00 2026-10-15T03:09:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:43.701+00:00 2026-10-15T03:09:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:44.706+00:00 2026-10-15T03:09:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:45.710+00:00 2026-10-15T03:09:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:46.719+00:00 2026-10-15T03:09:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:47.721+00:00 2026-10-15T03:09:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:48.726+00:00 2026-10-15T03:09:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:49.731+00:00 2026-10-15T03:09:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:50.736+00:00 2026-10-15T03:09:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:51.743+00:00 2026-10-15T03:09:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:52.747+00:00 2026-10-15T03:09:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:53.753+00:00 2026-10-15T03:09:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:54.757+00:00 2026-10-15T03:09:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:55.776+00:00 2026-10-15T03:09:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:56.771+00:00 2026-10-15T03:09:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:57.772+00:00 2026-10-15T03:09:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:58.778+00:00 2026-10-15T03:09:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:09:59.783+00:00 2026-10-15T03:09:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:00.787+00:00 2026-10-15T03:10:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:01.793+00:00 2026-10-15T03:10:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:02.799+00:00 2026-10-15T03:10:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:03.804+00:00 2026-10-15T03:10:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:04.809+00:00 2026-10-15T03:10:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:05.815+00:00 2026-10-15T03:10:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:06.819+00:00 2026-10-15T03:10:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:07.828+00:00 2026-10-15T03:10:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:08.829+00:00 2026-10-15T03:10:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:09.835+00:00 2026-10-15T03:10:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:10.845+00:00 2026-10-15T03:10:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:11.850+00:00 2026-10-15T03:10:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:12.851+00:00 2026-10-15T03:10:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:13.856+00:00 2026-10-15T03:10:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:14.862+00:00 2026-10-15T03:10:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:15.868+00:00 2026-10-15T03:10:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:16.872+00:00 2026-10-15T03:10:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:17.878+00:00 2026-10-15T03:10:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:18.884+00:00 2026-10-15T03:10:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:19.888+00:00 2026-10-15T03:10:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:20.982+00:00 2026-10-15T03:10:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:21.899+00:00 2026-10-15T03:10:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:22.904+00:00 2026-10-15T03:10:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:23.910+00:00 2026-10-15T03:10:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:24.914+00:00 2026-10-15T03:10:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:25.919+00:00 2026-10-15T03:10:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:26.934+00:00 2026-10-15T03:10:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:27.929+00:00 2026-10-15T03:10:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:28.937+00:00 2026-10-15T03:10:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:29.940+00:00 2026-10-15T03:10:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:30.946+00:00 2026-10-15T03:10:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:31.949+00:00 2026-10-15T03:10:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:32.955+00:00 2026-10-15T03:10:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:33.960+00:00 2026-10-15T03:10:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:34.966+00:00 2026-10-15T03:10:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:35.972+00:00 2026-10-15T03:10:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:36.976+00:00 2026-10-15T03:10:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:37.982+00:00 2026-10-15T03:10:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:38.987+00:00 2026-10-15T03:10:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:40.004+00:00 2026-10-15T03:10:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:40.999+00:00 2026-10-15T03:10:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:42.003+00:00 2026-10-15T03:10:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:43.009+00:00 2026-10-15T03:10:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:44.015+00:00 2026-10-15T03:10:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:45.029+00:00 2026-10-15T03:10:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:46.044+00:00 2026-10-15T03:10:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:47.030+00:00 2026-10-15T03:10:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:48.037+00:00 2026-10-15T03:10:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:49.040+00:00 2026-10-15T03:10:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:50.048+00:00 2026-10-15T03:10:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:51.050+00:00 2026-10-15T03:10:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:52.055+00:00 2026-10-15T03:10:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:53.063+00:00 2026-10-15T03:10:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:54.065+00:00 2026-10-15T03:10:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:55.071+00:00 2026-10-15T03:10:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:56.076+00:00 2026-10-15T03:10:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:57.081+00:00 2026-10-15T03:10:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:58.086+00:00 2026-10-15T03:10:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:10:59.091+00:00 2026-10-15T03:10:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:00.096+00:00 2026-10-15T03:11:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:01.109+00:00 2026-10-15T03:11:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:02.106+00:00 2026-10-15T03:11:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:03.111+00:00 2026-10-15T03:11:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:04.118+00:00 2026-10-15T03:11:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:05.123+00:00 2026-10-15T03:11:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:06.131+00:00 2026-10-15T03:11:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:07.138+00:00 2026-10-15T03:11:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:08.141+00:00 2026-10-15T03:11:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:09.180+00:00 2026-10-15T03:11:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:10.150+00:00 2026-10-15T03:11:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:11.213+00:00 2026-10-15T03:11:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:12.162+00:00 2026-10-15T03:11:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:13.164+00:00 2026-10-15T03:11:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:14.169+00:00 2026-10-15T03:11:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:15.175+00:00 2026-10-15T03:11:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:16.187+00:00 2026-10-15T03:11:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:17.185+00:00 2026-10-15T03:11:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:18.190+00:00 2026-10-15T03:11:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:19.196+00:00 2026-10-15T03:11:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:20.201+00:00 2026-10-15T03:11:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:21.206+00:00 2026-10-15T03:11:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:22.211+00:00 2026-10-15T03:11:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:23.217+00:00 2026-10-15T03:11:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:24.221+00:00 2026-10-15T03:11:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:25.226+00:00 2026-10-15T03:11:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:26.230+00:00 2026-10-15T03:11:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:27.236+00:00 2026-10-15T03:11:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:28.247+00:00 2026-10-15T03:11:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:29.246+00:00 2026-10-15T03:11:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:30.252+00:00 2026-10-15T03:11:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:31.256+00:00 2026-10-15T03:11:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:32.273+00:00 2026-10-15T03:11:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:33.272+00:00 2026-10-15T03:11:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:34.272+00:00 2026-10-15T03:11:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:35.277+00:00 2026-10-15T03:11:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:36.282+00:00 2026-10-15T03:11:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:37.298+00:00 2026-10-15T03:11:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:38.292+00:00 2026-10-15T03:11:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:39.298+00:00 2026-10-15T03:11:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:40.303+00:00 2026-10-15T03:11:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:41.308+00:00 2026-10-15T03:11:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:42.313+00:00 2026-10-15T03:11:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:43.318+00:00 2026-10-15T03:11:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:44.323+00:00 2026-10-15T03:11:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:45.329+00:00 2026-10-15T03:11:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:46.336+00:00 2026-10-15T03:11:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:47.357+00:00 2026-10-15T03:11:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:48.344+00:00 2026-10-15T03:11:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:49.386+00:00 2026-10-15T03:11:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:50.353+00:00 2026-10-15T03:11:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:51.358+00:00 2026-10-15T03:11:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:52.363+00:00 2026-10-15T03:11:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:53.376+00:00 2026-10-15T03:11:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:54.374+00:00 2026-10-15T03:11:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:55.378+00:00 2026-10-15T03:11:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:56.385+00:00 2026-10-15T03:11:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:57.389+00:00 2026-10-15T03:11:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:58.393+00:00 2026-10-15T03:11:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:11:59.398+00:00 2026-10-15T03:11:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:00.403+00:00 2026-10-15T03:12:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:01.491+00:00 2026-10-15T03:12:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:02.414+00:00 2026-10-15T03:12:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:03.417+00:00 2026-10-15T03:12:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:04.423+00:00 2026-10-15T03:12:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:05.427+00:00 2026-10-15T03:12:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:06.434+00:00 2026-10-15T03:12:06Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:06.434+00:00 2026-10-15T03:12:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:07.438+00:00 2026-10-15T03:12:07Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:07.438+00:00 2026-10-15T03:12:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:08.445+00:00 2026-10-15T03:12:08Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:08.445+00:00 2026-10-15T03:12:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:09.456+00:00 2026-10-15T03:12:09Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:09.456+00:00 2026-10-15T03:12:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:10.454+00:00 2026-10-15T03:12:10Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:10.454+00:00 2026-10-15T03:12:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:11.462+00:00 2026-10-15T03:12:11Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:11.462+00:00 2026-10-15T03:12:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:12.466+00:00 2026-10-15T03:12:12Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:12.466+00:00 2026-10-15T03:12:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:13.470+00:00 2026-10-15T03:12:13Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:12:13.470+00:00 2026-10-15T03:12:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log new file mode 100644 index 00000000..8a5d22c7 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T03:04:16.865+00:00 2026-10-15T03:04:16.851951680Z {"level":"info","ts":"2026-10-15T03:04:16Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"4c2fe683-2517-4a3a-b018-18a9782b7839","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} +2026-10-15T03:07:29.839+00:00 2026-10-15T03:07:29.833404182Z {"level":"info","ts":"2026-10-15T03:07:29Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"ed0ac0cd-45a6-4012-adaa-1fd004dac781","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json new file mode 100644 index 00000000..a61a4e27 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json @@ -0,0 +1,275 @@ +{ + "case": "A-0.5b-l4-cold", + "collected": "2026-10-15T03:38:49.515+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 03:04:15.891000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen2.5-0.5B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 03:04:15.892000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 03:07:22.617000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "ccc385bd-2c80-4be6-9ab7-0625542435ba" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 03:07:25.417000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 03:07:28.837000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 03:07:29.600000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 03:07:29.600000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 03:07:30.173000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 03:07:30.174000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 03:12:07.151000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4" + }, + "client.end": { + "ts": "2026-10-15 03:12:07.154000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 03:07:29+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 03:07:28.976000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 03:07:30.816000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 03:07:30.816000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 03:10:23+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 03:07:29+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 03:07:30+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 03:10:23+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 03:10:23+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 03:07:30+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 03:10:22+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 03:07:30+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "sampler.event_Pulling": { + "ts": "2026-10-15 03:07:33+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 03:10:22+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.566s (2m48.566s i", + "kubelet_pull_s": 168.566, + "pulled_from_cache": false + }, + "sampler.event_Created": { + "ts": "2026-10-15 03:10:22+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 03:10:22+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 03:10:33.926555+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 03:11:01.668050+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:01 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 03:11:03.371362+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:03 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 03:11:04.731338+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:04 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 03:11:06.644316+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:06 [default_loader.py:430] Loading weights took 1.92 seconds", + "self_reported_s": 1.92 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 03:11:07.445238+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:07 [model_runner.py:428] Model loading took 1.47 GiB memory and 4.387871 seconds", + "self_reported_s": 4.387871 + }, + "sampler.compile_done": { + "ts": "2026-10-15 03:11:38.154127+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:38 [monitor.py:53] torch.compile took 28.48 s in total", + "self_reported_s": 28.48 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 03:11:53.539804+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:53 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", + "self_reported_s": 11.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 03:11:54.078717+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:54 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 03:12:04.781237+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 03:12:04 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.33 s", + "self_reported_s": 57.33 + }, + "sampler.ready_key": { + "ts": "2026-10-15 03:12:06+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" + }, + "trainer": { + "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 03:04:16+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 03:04:16.578000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 03:04:16.578000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 03:04:18.224000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 03:07:11+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 03:04:16+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 03:04:17+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 03:07:11+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 03:07:11+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 03:04:17+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 03:07:10+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 03:04:17+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "trainer.event_Pulling": { + "ts": "2026-10-15 03:04:21+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 03:07:09+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.59s (2m48.59s inc", + "kubelet_pull_s": 168.59, + "pulled_from_cache": false + }, + "trainer.event_Created": { + "ts": "2026-10-15 03:07:09+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 03:07:10+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 03:07:17.253128+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 03:07:17.410020+00:00", + "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 03:07:21.686686+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 03:07:21.689422+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 03:07:21+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.md b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.md new file mode 100644 index 00000000..7b2007f6 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.md @@ -0,0 +1,111 @@ +# Cold start timeline: A-0.5b-l4-cold + +t=0 is the client's `create_model` call at 2026-10-15T03:04:15.892+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen2-5-0-5b-0-trainer` on `gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.workload_created | 03:04:16.000 | 0.1 | | Workload creationTimestamp (apiserver) | +| trainer.pod_created | 03:04:16.000 | 0.1 | | pod creationTimestamp (apiserver) | +| trainer.workload_first_seen_by_poller | 03:04:16.578 | 0.7 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 03:04:16.578 | 0.7 | | workloads.tsv poller: status.podName set | +| trainer.pod_Initialized | 03:04:17.000 | 1.1 | | pod condition Initialized | +| trainer.pod_PodScheduled | 03:04:17.000 | 1.1 | | pod condition PodScheduled | +| trainer.event_Scheduled | 03:04:17.000 | 1.1 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 | +| trainer.workload_reports_node | 03:04:18.224 | 2.3 | | workloads.tsv poller: status.nodeName set | +| trainer.event_Pulling | 03:04:21.000 | 5.1 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" | +| trainer.event_Pulled | 03:07:09.000 | 173.1 | 168.59 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" in 2m48.59s (2m48.59s inc | +| trainer.event_Created | 03:07:09.000 | 173.1 | | event: Container created | +| trainer.container_started | 03:07:10.000 | 174.1 | | containerStatuses.state.running.startedAt | +| trainer.event_Started | 03:07:10.000 | 174.1 | | event: Container started | +| trainer.condition_Placed_True | 03:07:11.000 | 175.1 | | Workload condition Placed | +| trainer.pod_Ready | 03:07:11.000 | 175.1 | | pod condition Ready | +| trainer.pod_ContainersReady | 03:07:11.000 | 175.1 | | pod condition ContainersReady | +| trainer.process_first_log | 03:07:17.253 | 181.4 | | container log, first line | +| trainer.load_start | 03:07:17.410 | 181.5 | | container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0... | +| trainer.ready_key | 03:07:21.000 | 185.1 | | redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B | +| trainer.load_done | 03:07:21.686 | 185.8 | | container log: Successfully loaded. | +| trainer.processor_started | 03:07:21.689 | 185.8 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1). | + +## sampler: `orw-lora-qwen-qwen2-5-0-5b-0-sampler` on `gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_first_seen_by_poller | 03:07:28.976 | 193.1 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_created | 03:07:29.000 | 193.1 | | Workload creationTimestamp (apiserver) | +| sampler.pod_created | 03:07:29.000 | 193.1 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 03:07:30.000 | 194.1 | | pod condition Initialized | +| sampler.pod_PodScheduled | 03:07:30.000 | 194.1 | | pod condition PodScheduled | +| sampler.event_Scheduled | 03:07:30.000 | 194.1 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 | +| sampler.workload_reports_pod | 03:07:30.816 | 194.9 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 03:07:30.816 | 194.9 | | workloads.tsv poller: status.nodeName set | +| sampler.event_Pulling | 03:07:33.000 | 197.1 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" | +| sampler.container_started | 03:10:22.000 | 366.1 | | containerStatuses.state.running.startedAt | +| sampler.event_Pulled | 03:10:22.000 | 366.1 | 168.566 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" in 2m48.566s (2m48.566s i | +| sampler.event_Created | 03:10:22.000 | 366.1 | | event: Container created | +| sampler.event_Started | 03:10:22.000 | 366.1 | | event: Container started | +| sampler.condition_Placed_True | 03:10:23.000 | 367.1 | | Workload condition Placed | +| sampler.pod_Ready | 03:10:23.000 | 367.1 | | pod condition Ready | +| sampler.pod_ContainersReady | 03:10:23.000 | 367.1 | | pod condition ContainersReady | +| sampler.process_first_log | 03:10:33.926 | 378.0 | | container log, first line | +| sampler.engine_init_start | 03:11:01.668 | 405.8 | | container log: (EngineCore pid=161) INFO 10-15 03:11:01 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 03:11:03.371 | 407.5 | | container log: (EngineCore pid=161) INFO 10-15 03:11:03 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 03:11:04.731 | 408.8 | | container log: (EngineCore pid=161) INFO 10-15 03:11:04 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 03:11:06.644 | 410.8 | 1.92 | container log: (EngineCore pid=161) INFO 10-15 03:11:06 [default_loader.py:430] Loading weights took 1.92 seconds | +| sampler.model_loaded | 03:11:07.445 | 411.6 | 4.387871 | container log: (EngineCore pid=161) INFO 10-15 03:11:07 [model_runner.py:428] Model loading took 1.47 GiB memory and 4.387871 seconds | +| sampler.compile_done | 03:11:38.154 | 442.3 | 28.48 | container log: (EngineCore pid=161) INFO 10-15 03:11:38 [monitor.py:53] torch.compile took 28.48 s in total | +| sampler.graphs_captured | 03:11:53.539 | 457.6 | 11.0 | container log: (EngineCore pid=161) INFO 10-15 03:11:53 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB | +| sampler.kv_cache_sized | 03:11:54.078 | 458.2 | | container log: (EngineCore pid=161) INFO 10-15 03:11:54 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB | +| sampler.engine_init_done | 03:12:04.781 | 468.9 | 57.33 | container log: (EngineCore pid=161) INFO 10-15 03:12:04 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.33 s | +| sampler.ready_key | 03:12:06.000 | 470.1 | | redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 03:04:15.891 | -0.0 | +| client.create_model.start | 03:04:15.892 | 0.0 | +| client.create_model.done | 03:07:22.617 | 186.7 | +| client.forward_backward.start | 03:07:25.417 | 189.5 | +| client.forward_backward.done | 03:07:28.837 | 192.9 | +| client.optim_step.done | 03:07:29.600 | 193.7 | +| client.save_weights_and_get_sampling_client.start | 03:07:29.600 | 193.7 | +| client.save_weights_and_get_sampling_client.done | 03:07:30.173 | 194.3 | +| client.sample.start | 03:07:30.174 | 194.3 | +| client.sample.done | 03:12:07.151 | 471.3 | +| client.end | 03:12:07.154 | 471.3 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.1 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 1.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 172.0 | +| container create | trainer.event_Pulled | trainer.container_started | 1.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.4 | +| weights load | trainer.load_start | trainer.load_done | 4.3 | +| ready key | trainer.load_done | trainer.ready_key | -0.7 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 186.7 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.4 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.6 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 1.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 172.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 11.9 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 27.7 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.7 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 3.3 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.5 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.6 | +| ready key | sampler.engine_init_done | sampler.ready_key | 1.2 | +| client: first sample blocked | client.sample.start | client.sample.done | 277.0 | + diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt new file mode 100644 index 00000000..3d7afe64 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T03:04:11.775+00:00 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt new file mode 100644 index 00000000..0d162c07 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json new file mode 100644 index 00000000..28c8d645 --- /dev/null +++ b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T03:07:29Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033823440495001", + "uid": "54443ec4-9700-416e-b5d7-846a4a007c00" + }, + "spec": { + "accelerator": { + "memory": "4Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "name": "worker", + "resources": { + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "d96cccda-acfd-4fa4-95c5-72c958a83f6b", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T03:10:23Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "4Gi", + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T03:04:16Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033631210927015", + "uid": "47b54db3-9dc6-4dfd-a972-55bee1fc0dad" + }, + "spec": { + "accelerator": { + "memory": "5Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "name": "worker", + "resources": { + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "33cc03f0-42b9-4860-8d5e-f14ca74f3511", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T03:07:11Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "5Gi", + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log new file mode 100644 index 00000000..e3ebe340 --- /dev/null +++ b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log @@ -0,0 +1,31 @@ +2026-10-15T02:59:03.962+00:00 2026-10-15T02:59:03.964951462Z INFO: 10.32.3.1:54528 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T02:59:11.901+00:00 2026-10-15T02:59:11.904243760Z INFO: 127.0.0.1:55772 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T02:59:12.147+00:00 2026-10-15T02:59:12.151769067Z INFO: 127.0.0.1:55782 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T02:59:12.211+00:00 2026-10-15T02:59:12.215619037Z INFO: 127.0.0.1:55782 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T02:59:12.479+00:00 2026-10-15T02:59:12.482819930Z [API_SERVER] enqueued op=create_model request_id=cc688d59-8147-4315-9236-81ea9296fa1d model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T02:59:12.481+00:00 2026-10-15T02:59:12.483378195Z INFO: 127.0.0.1:55792 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T02:59:13.961+00:00 2026-10-15T02:59:13.965191360Z INFO: 10.32.3.1:48124 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T02:59:22.388+00:00 2026-10-15T02:59:22.390801550Z INFO: 127.0.0.1:45904 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T02:59:23.961+00:00 2026-10-15T02:59:23.964632011Z INFO: 10.32.3.1:39380 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T02:59:29.361+00:00 2026-10-15T02:59:29.363125454Z INFO: 127.0.0.1:45914 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T02:59:32.441+00:00 2026-10-15T02:59:32.444410717Z [API_SERVER] enqueued op=forward_backward request_id=0aefaadb-3d41-4420-89c5-bf8de12892b6 model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T02:59:32.441+00:00 2026-10-15T02:59:32.445344260Z INFO: 127.0.0.1:45914 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T02:59:33.961+00:00 2026-10-15T02:59:33.964713101Z INFO: 10.32.3.1:51494 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T02:59:35.560+00:00 2026-10-15T02:59:35.564042664Z [API_SERVER] enqueued op=optim_step request_id=cf94adcb-6e7f-47ac-97bc-f5de0168c89b model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T02:59:35.561+00:00 2026-10-15T02:59:35.564697067Z INFO: 127.0.0.1:45914 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T02:59:36.462+00:00 2026-10-15T02:59:36.445520836Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=d5021fb2-234b-470a-b272-1ad3d13c1ef6 model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T02:59:36.462+00:00 2026-10-15T02:59:36.446145534Z INFO: 127.0.0.1:45914 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T02:59:37.136+00:00 2026-10-15T02:59:37.140404264Z INFO: 127.0.0.1:59990 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T02:59:43.962+00:00 2026-10-15T02:59:43.964864449Z INFO: 10.32.3.1:53026 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T02:59:53.962+00:00 2026-10-15T02:59:53.965184980Z INFO: 10.32.3.1:58910 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:00:03.962+00:00 2026-10-15T03:00:03.964285482Z INFO: 10.32.3.1:44748 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:00:13.962+00:00 2026-10-15T03:00:13.964568310Z INFO: 10.32.3.1:55124 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:00:22.589+00:00 2026-10-15T03:00:22.590580301Z INFO: 127.0.0.1:41576 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:00:23.962+00:00 2026-10-15T03:00:23.965334932Z INFO: 10.32.3.1:37274 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:00:33.962+00:00 2026-10-15T03:00:33.965008993Z INFO: 10.32.3.1:48328 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:00:43.962+00:00 2026-10-15T03:00:43.965137758Z INFO: 10.32.3.1:57986 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:00:53.962+00:00 2026-10-15T03:00:53.964797296Z INFO: 10.32.3.1:35412 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:01:03.965+00:00 2026-10-15T03:01:03.965183315Z INFO: 10.32.3.1:41724 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:01:12.798+00:00 2026-10-15T03:01:12.794180811Z INFO: 127.0.0.1:45096 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:01:13.967+00:00 2026-10-15T03:01:13.964671642Z INFO: 10.32.3.1:51088 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:01:23.978+00:00 2026-10-15T03:01:23.965052643Z INFO: 10.32.3.1:42154 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl new file mode 100644 index 00000000..8d9562a1 --- /dev/null +++ b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T02:59:11.608+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T02:59:11.609+00:00", "t": 0.001, "phase": "create_model.start"} +{"ts": "2026-10-15T02:59:29.165+00:00", "t": 17.557, "phase": "create_model.done", "model_id": "cc688d59-8147-4315-9236-81ea9296fa1d"} +{"ts": "2026-10-15T02:59:32.362+00:00", "t": 20.753, "phase": "forward_backward.start"} +{"ts": "2026-10-15T02:59:35.480+00:00", "t": 23.872, "phase": "forward_backward.done"} +{"ts": "2026-10-15T02:59:36.339+00:00", "t": 24.731, "phase": "optim_step.done"} +{"ts": "2026-10-15T02:59:36.340+00:00", "t": 24.732, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T02:59:36.946+00:00", "t": 25.338, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T02:59:36.947+00:00", "t": 25.339, "phase": "sample.start"} +{"ts": "2026-10-15T03:01:23.638+00:00", "t": 132.029, "phase": "sample.done", "text": " 4"} +{"ts": "2026-10-15T03:01:23.639+00:00", "t": 132.03, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log new file mode 100644 index 00000000..ebf20190 --- /dev/null +++ b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log @@ -0,0 +1,16 @@ + Building open-rl-client @ file:///Users/sunilarora/work/open-rl/examples + Built open-rl-client @ file:///Users/sunilarora/work/open-rl/examples +Uninstalled 1 package in 1ms +Installed 1 package in 4ms +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.001s create_model.start +[cold-start] 17.557s create_model.done {'model_id': 'cc688d59-8147-4315-9236-81ea9296fa1d'} +[cold-start] 20.753s forward_backward.start +[cold-start] 23.872s forward_backward.done +[cold-start] 24.731s optim_step.done +[cold-start] 24.732s save_weights_and_get_sampling_client.start +[cold-start] 25.338s save_weights_and_get_sampling_client.done +[cold-start] 25.339s sample.start +[cold-start] 132.029s sample.done {'text': ' 4'} +[cold-start] 132.030s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt new file mode 100644 index 00000000..eed48c3c --- /dev/null +++ b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T03:01:46.957+00:00 diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json new file mode 100644 index 00000000..b5ad8cd7 --- /dev/null +++ b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json @@ -0,0 +1,269 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:36Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176697983001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:36Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:36Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94ae008553cd", + "namespace": "openrl-system", + "resourceVersion": "1792033176815007016", + "uid": "363bd5d3-147c-4b0d-8500-d3a3b6232789" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:40Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176788543001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:40Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:40Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed139d233", + "namespace": "openrl-system", + "resourceVersion": "1792033180304687014", + "uid": "2463b9b8-32de-4c33-af86-ca945ab359e3" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:40Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176788543001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:40Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:40Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed2f06544", + "namespace": "openrl-system", + "resourceVersion": "1792033180345279015", + "uid": "4766979d-5fb4-4934-96f7-b8e5c1833efa" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:40Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792033176788543001", + "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:40Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:40Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aedddd6086", + "namespace": "openrl-system", + "resourceVersion": "1792033180517807014", + "uid": "e1c7e13f-7b29-4ed9-afdb-cc1a062a32d2" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153094671012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a883fadef0", + "namespace": "openrl-system", + "resourceVersion": "1792033153265231011", + "uid": "62c5cae6-2deb-43a0-a097-73157e760539" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:17Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153224047012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:17Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:17Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a9671f24f0", + "namespace": "openrl-system", + "resourceVersion": "1792033157084143008", + "uid": "9472795b-31b9-478f-a545-6146795d5503" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:17Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153224047012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:17Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:17Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a968b7d635", + "namespace": "openrl-system", + "resourceVersion": "1792033157103615024", + "uid": "11d1cf36-8464-42c3-94e2-1843723223c8" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T02:59:17Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792033153224047012", + "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T02:59:17Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T02:59:17Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a973285d70", + "namespace": "openrl-system", + "resourceVersion": "1792033157264271007", + "uid": "8fa654a4-608f-4e8c-804f-9d1d5c91650a" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt new file mode 100644 index 00000000..ba103934 --- /dev/null +++ b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt @@ -0,0 +1,120 @@ +2026-10-15T02:59:51.481521360Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T03:00:03.567266539Z INFO 10-15 03:00:03 [model.py:692] Resolved architecture: Qwen2ForCausalLM +2026-10-15T03:00:03.567589212Z INFO 10-15 03:00:03 [model.py:2030] Using max model len 8192 +2026-10-15T03:00:03.569678337Z INFO 10-15 03:00:03 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. +2026-10-15T03:00:03.982387755Z INFO 10-15 03:00:03 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T03:00:07.254113521Z WARNING 10-15 03:00:07 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T03:00:18.656636083Z (EngineCore pid=161) INFO 10-15 03:00:18 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T03:00:19.465208619Z (EngineCore pid=161) INFO 10-15 03:00:19 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_83c4163e04374cbaaa5cdd2b203e669a backend=nccl +2026-10-15T03:00:19.472365440Z (EngineCore pid=161) INFO 10-15 03:00:19 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T03:00:19.472394618Z (EngineCore pid=161) INFO 10-15 03:00:19 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T03:00:20.368364392Z (EngineCore pid=161) INFO 10-15 03:00:20 [model_runner.py:396] Loading model from scratch... +2026-10-15T03:00:21.228539711Z (EngineCore pid=161) INFO 10-15 03:00:21 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T03:00:21.228925613Z (EngineCore pid=161) INFO 10-15 03:00:21 [flash_attn.py:1116] Using FlashAttention version 2 +2026-10-15T03:00:21.343550869Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T03:00:21.343876865Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T03:00:21.715756916Z (EngineCore pid=161) INFO 10-15 03:00:21 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.61 GiB. +2026-10-15T03:00:21.715792572Z (EngineCore pid=161) INFO 10-15 03:00:21 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T03:00:21.716786271Z (EngineCore pid=161) +Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T02:59:23.867929492Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T02:59:23.867951058Z +2026-10-15T02:59:23.989922239Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... +2026-10-15T02:59:24.254380821Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T02:59:28.134128443Z +Loading weights: 0%| | 0/290 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T03:33:16.441826067Z (EngineCore pid=168) INFO 10-15 03:33:16 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c6e1717f2b3f4f1aad5a35495f331a54 backend=nccl +2026-10-15T03:33:16.446140127Z (EngineCore pid=168) INFO 10-15 03:33:16 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T03:33:16.446146945Z (EngineCore pid=168) INFO 10-15 03:33:16 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T03:33:17.209275505Z (EngineCore pid=168) INFO 10-15 03:33:17 [model_runner.py:396] Loading model from scratch... +2026-10-15T03:33:17.782392712Z (EngineCore pid=168) INFO 10-15 03:33:17 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T03:33:17.782713265Z (EngineCore pid=168) INFO 10-15 03:33:17 [flash_attn.py:1116] Using FlashAttention version 3 +2026-10-15T03:33:17.897956609Z (EngineCore pid=168) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T03:33:17.898105102Z (EngineCore pid=168) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T03:33:18.247383946Z (EngineCore pid=168) INFO 10-15 03:33:18 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. +2026-10-15T03:33:18.247405710Z (EngineCore pid=168) INFO 10-15 03:33:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T03:33:18.247843823Z (EngineCore pid=168) +Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T03:15:56.032450989Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T03:15:56.032474492Z +2026-10-15T03:15:56.267417499Z Loading base model Qwen/Qwen3-8B to cuda:0... +2026-10-15T03:15:56.394801806Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T03:30:59.475214316Z +Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 24h v1.35.8-gke.1380001 true +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 24h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json b/runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json new file mode 100644 index 00000000..e1e02eb8 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-api-server-645ff9777d-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "645ff9777d", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + } + ], + "resourceVersion": "1791941173996783021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-79xdl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:02Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:26:02Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.10", + "podIPs": [ + { + "ip": "10.32.3.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T03:31:11Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-sampler", + "uid": "4637c9e3-a819-432d-9edb-58a1522564c4" + } + ], + "resourceVersion": "1792035171317391023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-sampler-4637c9e3" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "464502aa-adbd-4913-8de9-9cf72fa2e24d" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-4dt6k", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-4dt6k", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:32:51Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:31:11Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:32:51Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:32:51Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:31:11Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "40Gi" + }, + "containerID": "containerd://3427c91c85176d6732b6c0d7374ca8c171ff0233228cb15a8abf662d4ddfe519", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T03:32:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-4dt6k", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.1.7", + "podIPs": [ + { + "ip": "10.32.1.7" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T03:31:11Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T03:14:14Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-trainer", + "uid": "013a69b1-c398-4e81-a5c5-90edf5da837a" + } + ], + "resourceVersion": "1792034152554671006", + "uid": "5f7199d4-14aa-4747-9728-d940f5af679f" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-trainer-013a69b1" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "bbe9937b-9591-4fe5-9cd5-1845592a917d" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-qss8v", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-qss8v", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:15:52Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:14:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:15:52Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:15:52Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T03:14:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "36Gi" + }, + "containerID": "containerd://40c14afec9852a4e978707988d4e5ae0ddd5abd2a58fd807ce3b8e89eefe6dc6", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T03:15:51Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-qss8v", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.4.6", + "podIPs": [ + { + "ip": "10.32.4.6" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T03:14:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log b/runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log new file mode 100644 index 00000000..e9451fdb --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log @@ -0,0 +1,1734 @@ +2026-10-15T03:14:10.732+00:00 2026-10-15T03:14:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:11.737+00:00 2026-10-15T03:14:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:12.741+00:00 2026-10-15T03:14:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:13.746+00:00 2026-10-15T03:14:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:14.751+00:00 2026-10-15T03:14:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:15.757+00:00 2026-10-15T03:14:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:16.762+00:00 2026-10-15T03:14:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:17.766+00:00 2026-10-15T03:14:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:18.772+00:00 2026-10-15T03:14:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:19.779+00:00 2026-10-15T03:14:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:20.782+00:00 2026-10-15T03:14:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:21.787+00:00 2026-10-15T03:14:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:22.792+00:00 2026-10-15T03:14:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:23.797+00:00 2026-10-15T03:14:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:24.806+00:00 2026-10-15T03:14:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:25.807+00:00 2026-10-15T03:14:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:26.813+00:00 2026-10-15T03:14:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:27.816+00:00 2026-10-15T03:14:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:28.823+00:00 2026-10-15T03:14:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:29.837+00:00 2026-10-15T03:14:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:30.833+00:00 2026-10-15T03:14:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:31.838+00:00 2026-10-15T03:14:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:32.843+00:00 2026-10-15T03:14:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:33.848+00:00 2026-10-15T03:14:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:34.855+00:00 2026-10-15T03:14:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:35.860+00:00 2026-10-15T03:14:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:36.865+00:00 2026-10-15T03:14:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:37.870+00:00 2026-10-15T03:14:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:38.876+00:00 2026-10-15T03:14:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:39.883+00:00 2026-10-15T03:14:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:40.887+00:00 2026-10-15T03:14:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:41.893+00:00 2026-10-15T03:14:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:42.898+00:00 2026-10-15T03:14:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:43.903+00:00 2026-10-15T03:14:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:44.929+00:00 2026-10-15T03:14:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:45.914+00:00 2026-10-15T03:14:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:46.919+00:00 2026-10-15T03:14:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:47.924+00:00 2026-10-15T03:14:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:48.929+00:00 2026-10-15T03:14:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:50.075+00:00 2026-10-15T03:14:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:50.940+00:00 2026-10-15T03:14:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:51.946+00:00 2026-10-15T03:14:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:52.951+00:00 2026-10-15T03:14:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:53.956+00:00 2026-10-15T03:14:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:54.976+00:00 2026-10-15T03:14:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:55.969+00:00 2026-10-15T03:14:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:56.975+00:00 2026-10-15T03:14:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:57.981+00:00 2026-10-15T03:14:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:58.986+00:00 2026-10-15T03:14:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:14:59.990+00:00 2026-10-15T03:14:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:00.996+00:00 2026-10-15T03:15:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:02.003+00:00 2026-10-15T03:15:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:03.009+00:00 2026-10-15T03:15:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:04.014+00:00 2026-10-15T03:15:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:05.020+00:00 2026-10-15T03:15:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:06.026+00:00 2026-10-15T03:15:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:07.032+00:00 2026-10-15T03:15:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:08.038+00:00 2026-10-15T03:15:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:09.044+00:00 2026-10-15T03:15:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:10.051+00:00 2026-10-15T03:15:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:11.058+00:00 2026-10-15T03:15:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:12.062+00:00 2026-10-15T03:15:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:13.069+00:00 2026-10-15T03:15:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:14.090+00:00 2026-10-15T03:15:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:15.081+00:00 2026-10-15T03:15:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:16.084+00:00 2026-10-15T03:15:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:17.089+00:00 2026-10-15T03:15:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:18.131+00:00 2026-10-15T03:15:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:19.100+00:00 2026-10-15T03:15:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:20.106+00:00 2026-10-15T03:15:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:21.112+00:00 2026-10-15T03:15:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:22.118+00:00 2026-10-15T03:15:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:23.141+00:00 2026-10-15T03:15:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:24.130+00:00 2026-10-15T03:15:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:25.139+00:00 2026-10-15T03:15:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:26.143+00:00 2026-10-15T03:15:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:27.149+00:00 2026-10-15T03:15:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:28.166+00:00 2026-10-15T03:15:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:29.160+00:00 2026-10-15T03:15:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:30.166+00:00 2026-10-15T03:15:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:31.215+00:00 2026-10-15T03:15:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:32.177+00:00 2026-10-15T03:15:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:33.183+00:00 2026-10-15T03:15:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:34.188+00:00 2026-10-15T03:15:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:35.194+00:00 2026-10-15T03:15:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:36.201+00:00 2026-10-15T03:15:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:37.205+00:00 2026-10-15T03:15:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:38.211+00:00 2026-10-15T03:15:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:39.215+00:00 2026-10-15T03:15:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:40.221+00:00 2026-10-15T03:15:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:41.227+00:00 2026-10-15T03:15:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:42.232+00:00 2026-10-15T03:15:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:43.237+00:00 2026-10-15T03:15:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:44.243+00:00 2026-10-15T03:15:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:45.248+00:00 2026-10-15T03:15:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:46.254+00:00 2026-10-15T03:15:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:47.258+00:00 2026-10-15T03:15:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:48.264+00:00 2026-10-15T03:15:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:49.269+00:00 2026-10-15T03:15:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:50.279+00:00 2026-10-15T03:15:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:51.280+00:00 2026-10-15T03:15:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:52.285+00:00 2026-10-15T03:15:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:53.290+00:00 2026-10-15T03:15:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:54.297+00:00 2026-10-15T03:15:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:55.301+00:00 2026-10-15T03:15:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:56.306+00:00 2026-10-15T03:15:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:57.313+00:00 2026-10-15T03:15:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:58.320+00:00 2026-10-15T03:15:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:15:59.324+00:00 2026-10-15T03:15:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:00.327+00:00 2026-10-15T03:16:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:01.334+00:00 2026-10-15T03:16:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:02.340+00:00 2026-10-15T03:16:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:03.346+00:00 2026-10-15T03:16:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:04.351+00:00 2026-10-15T03:16:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:05.357+00:00 2026-10-15T03:16:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:06.361+00:00 2026-10-15T03:16:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:07.368+00:00 2026-10-15T03:16:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:08.372+00:00 2026-10-15T03:16:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:09.400+00:00 2026-10-15T03:16:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:10.382+00:00 2026-10-15T03:16:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:11.387+00:00 2026-10-15T03:16:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:12.392+00:00 2026-10-15T03:16:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:13.397+00:00 2026-10-15T03:16:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:14.402+00:00 2026-10-15T03:16:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:15.407+00:00 2026-10-15T03:16:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:16.413+00:00 2026-10-15T03:16:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:17.418+00:00 2026-10-15T03:16:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:18.422+00:00 2026-10-15T03:16:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:19.428+00:00 2026-10-15T03:16:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:20.433+00:00 2026-10-15T03:16:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:21.438+00:00 2026-10-15T03:16:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:22.444+00:00 2026-10-15T03:16:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:23.448+00:00 2026-10-15T03:16:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:24.454+00:00 2026-10-15T03:16:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:25.512+00:00 2026-10-15T03:16:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:26.466+00:00 2026-10-15T03:16:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:27.558+00:00 2026-10-15T03:16:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:28.481+00:00 2026-10-15T03:16:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:29.480+00:00 2026-10-15T03:16:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:30.485+00:00 2026-10-15T03:16:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:31.491+00:00 2026-10-15T03:16:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:32.498+00:00 2026-10-15T03:16:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:33.502+00:00 2026-10-15T03:16:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:34.508+00:00 2026-10-15T03:16:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:35.513+00:00 2026-10-15T03:16:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:36.525+00:00 2026-10-15T03:16:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:37.527+00:00 2026-10-15T03:16:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:38.529+00:00 2026-10-15T03:16:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:39.535+00:00 2026-10-15T03:16:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:40.539+00:00 2026-10-15T03:16:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:41.545+00:00 2026-10-15T03:16:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:42.551+00:00 2026-10-15T03:16:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:43.556+00:00 2026-10-15T03:16:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:44.562+00:00 2026-10-15T03:16:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:45.568+00:00 2026-10-15T03:16:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:46.574+00:00 2026-10-15T03:16:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:47.581+00:00 2026-10-15T03:16:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:48.594+00:00 2026-10-15T03:16:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:49.605+00:00 2026-10-15T03:16:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:50.595+00:00 2026-10-15T03:16:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:51.600+00:00 2026-10-15T03:16:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:52.612+00:00 2026-10-15T03:16:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:53.610+00:00 2026-10-15T03:16:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:54.617+00:00 2026-10-15T03:16:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:55.623+00:00 2026-10-15T03:16:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:56.629+00:00 2026-10-15T03:16:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:57.634+00:00 2026-10-15T03:16:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:58.639+00:00 2026-10-15T03:16:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:16:59.645+00:00 2026-10-15T03:16:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:00.650+00:00 2026-10-15T03:17:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:01.656+00:00 2026-10-15T03:17:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:02.661+00:00 2026-10-15T03:17:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:03.667+00:00 2026-10-15T03:17:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:04.673+00:00 2026-10-15T03:17:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:05.678+00:00 2026-10-15T03:17:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:06.683+00:00 2026-10-15T03:17:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:07.688+00:00 2026-10-15T03:17:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:08.693+00:00 2026-10-15T03:17:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:09.698+00:00 2026-10-15T03:17:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:10.703+00:00 2026-10-15T03:17:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:11.719+00:00 2026-10-15T03:17:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:12.714+00:00 2026-10-15T03:17:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:13.722+00:00 2026-10-15T03:17:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:14.738+00:00 2026-10-15T03:17:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:15.745+00:00 2026-10-15T03:17:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:16.736+00:00 2026-10-15T03:17:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:17.741+00:00 2026-10-15T03:17:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:18.746+00:00 2026-10-15T03:17:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:19.754+00:00 2026-10-15T03:17:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:20.758+00:00 2026-10-15T03:17:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:21.764+00:00 2026-10-15T03:17:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:22.768+00:00 2026-10-15T03:17:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:23.773+00:00 2026-10-15T03:17:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:24.778+00:00 2026-10-15T03:17:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:25.784+00:00 2026-10-15T03:17:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:26.789+00:00 2026-10-15T03:17:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:27.793+00:00 2026-10-15T03:17:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:28.801+00:00 2026-10-15T03:17:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:29.804+00:00 2026-10-15T03:17:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:30.810+00:00 2026-10-15T03:17:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:31.815+00:00 2026-10-15T03:17:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:32.822+00:00 2026-10-15T03:17:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:33.827+00:00 2026-10-15T03:17:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:34.835+00:00 2026-10-15T03:17:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:35.840+00:00 2026-10-15T03:17:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:36.845+00:00 2026-10-15T03:17:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:37.852+00:00 2026-10-15T03:17:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:38.857+00:00 2026-10-15T03:17:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:39.864+00:00 2026-10-15T03:17:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:40.870+00:00 2026-10-15T03:17:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:41.876+00:00 2026-10-15T03:17:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:42.889+00:00 2026-10-15T03:17:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:43.888+00:00 2026-10-15T03:17:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:44.894+00:00 2026-10-15T03:17:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:45.907+00:00 2026-10-15T03:17:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:46.904+00:00 2026-10-15T03:17:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:47.909+00:00 2026-10-15T03:17:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:48.914+00:00 2026-10-15T03:17:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:49.919+00:00 2026-10-15T03:17:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:50.925+00:00 2026-10-15T03:17:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:51.930+00:00 2026-10-15T03:17:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:52.963+00:00 2026-10-15T03:17:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:53.942+00:00 2026-10-15T03:17:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:54.947+00:00 2026-10-15T03:17:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:55.954+00:00 2026-10-15T03:17:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:56.956+00:00 2026-10-15T03:17:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:57.962+00:00 2026-10-15T03:17:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:58.966+00:00 2026-10-15T03:17:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:17:59.970+00:00 2026-10-15T03:17:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:00.985+00:00 2026-10-15T03:18:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:01.980+00:00 2026-10-15T03:18:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:02.986+00:00 2026-10-15T03:18:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:03.991+00:00 2026-10-15T03:18:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:05.001+00:00 2026-10-15T03:18:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:06.090+00:00 2026-10-15T03:18:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:07.008+00:00 2026-10-15T03:18:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:08.014+00:00 2026-10-15T03:18:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:09.019+00:00 2026-10-15T03:18:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:10.025+00:00 2026-10-15T03:18:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:11.029+00:00 2026-10-15T03:18:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:12.034+00:00 2026-10-15T03:18:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:13.039+00:00 2026-10-15T03:18:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:14.043+00:00 2026-10-15T03:18:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:15.049+00:00 2026-10-15T03:18:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:16.053+00:00 2026-10-15T03:18:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:17.058+00:00 2026-10-15T03:18:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:18.064+00:00 2026-10-15T03:18:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:19.071+00:00 2026-10-15T03:18:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:20.073+00:00 2026-10-15T03:18:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:21.079+00:00 2026-10-15T03:18:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:22.086+00:00 2026-10-15T03:18:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:23.091+00:00 2026-10-15T03:18:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:24.096+00:00 2026-10-15T03:18:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:25.107+00:00 2026-10-15T03:18:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:26.112+00:00 2026-10-15T03:18:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:27.116+00:00 2026-10-15T03:18:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:28.120+00:00 2026-10-15T03:18:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:29.124+00:00 2026-10-15T03:18:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:30.129+00:00 2026-10-15T03:18:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:31.135+00:00 2026-10-15T03:18:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:32.141+00:00 2026-10-15T03:18:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:33.186+00:00 2026-10-15T03:18:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:34.151+00:00 2026-10-15T03:18:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:35.157+00:00 2026-10-15T03:18:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:36.162+00:00 2026-10-15T03:18:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:37.167+00:00 2026-10-15T03:18:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:38.172+00:00 2026-10-15T03:18:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:39.183+00:00 2026-10-15T03:18:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:40.183+00:00 2026-10-15T03:18:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:41.187+00:00 2026-10-15T03:18:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:42.193+00:00 2026-10-15T03:18:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:43.198+00:00 2026-10-15T03:18:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:44.203+00:00 2026-10-15T03:18:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:45.207+00:00 2026-10-15T03:18:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:46.212+00:00 2026-10-15T03:18:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:47.217+00:00 2026-10-15T03:18:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:48.268+00:00 2026-10-15T03:18:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:49.228+00:00 2026-10-15T03:18:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:50.232+00:00 2026-10-15T03:18:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:51.237+00:00 2026-10-15T03:18:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:52.242+00:00 2026-10-15T03:18:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:53.247+00:00 2026-10-15T03:18:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:54.254+00:00 2026-10-15T03:18:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:55.258+00:00 2026-10-15T03:18:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:56.264+00:00 2026-10-15T03:18:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:57.268+00:00 2026-10-15T03:18:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:58.273+00:00 2026-10-15T03:18:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:18:59.279+00:00 2026-10-15T03:18:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:00.283+00:00 2026-10-15T03:19:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:01.288+00:00 2026-10-15T03:19:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:02.298+00:00 2026-10-15T03:19:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:03.299+00:00 2026-10-15T03:19:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:04.304+00:00 2026-10-15T03:19:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:05.308+00:00 2026-10-15T03:19:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:06.313+00:00 2026-10-15T03:19:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:07.318+00:00 2026-10-15T03:19:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:08.325+00:00 2026-10-15T03:19:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:09.329+00:00 2026-10-15T03:19:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:10.334+00:00 2026-10-15T03:19:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:11.341+00:00 2026-10-15T03:19:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:12.345+00:00 2026-10-15T03:19:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:13.350+00:00 2026-10-15T03:19:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:14.356+00:00 2026-10-15T03:19:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:15.363+00:00 2026-10-15T03:19:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:16.367+00:00 2026-10-15T03:19:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:17.371+00:00 2026-10-15T03:19:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:18.377+00:00 2026-10-15T03:19:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:19.382+00:00 2026-10-15T03:19:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:20.389+00:00 2026-10-15T03:19:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:21.395+00:00 2026-10-15T03:19:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:22.399+00:00 2026-10-15T03:19:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:23.404+00:00 2026-10-15T03:19:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:24.409+00:00 2026-10-15T03:19:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:25.414+00:00 2026-10-15T03:19:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:26.419+00:00 2026-10-15T03:19:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:27.424+00:00 2026-10-15T03:19:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:28.430+00:00 2026-10-15T03:19:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:29.437+00:00 2026-10-15T03:19:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:30.440+00:00 2026-10-15T03:19:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:31.445+00:00 2026-10-15T03:19:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:32.451+00:00 2026-10-15T03:19:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:33.459+00:00 2026-10-15T03:19:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:34.470+00:00 2026-10-15T03:19:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:35.469+00:00 2026-10-15T03:19:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:36.478+00:00 2026-10-15T03:19:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:37.483+00:00 2026-10-15T03:19:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:38.489+00:00 2026-10-15T03:19:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:39.491+00:00 2026-10-15T03:19:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:40.497+00:00 2026-10-15T03:19:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:41.501+00:00 2026-10-15T03:19:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:42.508+00:00 2026-10-15T03:19:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:43.533+00:00 2026-10-15T03:19:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:44.519+00:00 2026-10-15T03:19:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:45.523+00:00 2026-10-15T03:19:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:46.528+00:00 2026-10-15T03:19:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:47.533+00:00 2026-10-15T03:19:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:48.539+00:00 2026-10-15T03:19:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:49.544+00:00 2026-10-15T03:19:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:50.549+00:00 2026-10-15T03:19:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:51.554+00:00 2026-10-15T03:19:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:52.559+00:00 2026-10-15T03:19:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:53.564+00:00 2026-10-15T03:19:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:54.570+00:00 2026-10-15T03:19:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:55.575+00:00 2026-10-15T03:19:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:56.580+00:00 2026-10-15T03:19:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:57.584+00:00 2026-10-15T03:19:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:58.595+00:00 2026-10-15T03:19:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:19:59.594+00:00 2026-10-15T03:19:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:00.599+00:00 2026-10-15T03:20:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:01.611+00:00 2026-10-15T03:20:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:02.609+00:00 2026-10-15T03:20:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:03.615+00:00 2026-10-15T03:20:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:04.620+00:00 2026-10-15T03:20:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:05.642+00:00 2026-10-15T03:20:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:06.631+00:00 2026-10-15T03:20:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:07.634+00:00 2026-10-15T03:20:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:08.638+00:00 2026-10-15T03:20:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:09.643+00:00 2026-10-15T03:20:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:10.651+00:00 2026-10-15T03:20:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:11.653+00:00 2026-10-15T03:20:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:12.658+00:00 2026-10-15T03:20:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:13.663+00:00 2026-10-15T03:20:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:14.668+00:00 2026-10-15T03:20:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:15.674+00:00 2026-10-15T03:20:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:16.678+00:00 2026-10-15T03:20:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:17.683+00:00 2026-10-15T03:20:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:18.688+00:00 2026-10-15T03:20:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:19.693+00:00 2026-10-15T03:20:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:20.705+00:00 2026-10-15T03:20:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:21.712+00:00 2026-10-15T03:20:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:22.707+00:00 2026-10-15T03:20:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:23.712+00:00 2026-10-15T03:20:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:24.718+00:00 2026-10-15T03:20:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:25.721+00:00 2026-10-15T03:20:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:26.726+00:00 2026-10-15T03:20:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:27.731+00:00 2026-10-15T03:20:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:28.742+00:00 2026-10-15T03:20:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:29.742+00:00 2026-10-15T03:20:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:30.748+00:00 2026-10-15T03:20:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:31.751+00:00 2026-10-15T03:20:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:32.756+00:00 2026-10-15T03:20:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:33.762+00:00 2026-10-15T03:20:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:34.766+00:00 2026-10-15T03:20:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:35.787+00:00 2026-10-15T03:20:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:36.776+00:00 2026-10-15T03:20:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:37.780+00:00 2026-10-15T03:20:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:38.791+00:00 2026-10-15T03:20:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:39.790+00:00 2026-10-15T03:20:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:40.796+00:00 2026-10-15T03:20:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:41.800+00:00 2026-10-15T03:20:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:42.804+00:00 2026-10-15T03:20:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:43.809+00:00 2026-10-15T03:20:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:44.814+00:00 2026-10-15T03:20:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:45.819+00:00 2026-10-15T03:20:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:46.833+00:00 2026-10-15T03:20:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:47.830+00:00 2026-10-15T03:20:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:48.836+00:00 2026-10-15T03:20:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:49.846+00:00 2026-10-15T03:20:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:50.846+00:00 2026-10-15T03:20:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:51.850+00:00 2026-10-15T03:20:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:52.855+00:00 2026-10-15T03:20:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:53.860+00:00 2026-10-15T03:20:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:54.874+00:00 2026-10-15T03:20:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:55.870+00:00 2026-10-15T03:20:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:56.912+00:00 2026-10-15T03:20:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:57.881+00:00 2026-10-15T03:20:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:58.891+00:00 2026-10-15T03:20:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:20:59.891+00:00 2026-10-15T03:20:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:00.896+00:00 2026-10-15T03:21:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:01.901+00:00 2026-10-15T03:21:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:02.907+00:00 2026-10-15T03:21:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:03.912+00:00 2026-10-15T03:21:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:04.921+00:00 2026-10-15T03:21:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:05.926+00:00 2026-10-15T03:21:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:06.934+00:00 2026-10-15T03:21:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:07.938+00:00 2026-10-15T03:21:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:08.951+00:00 2026-10-15T03:21:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:09.948+00:00 2026-10-15T03:21:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:10.953+00:00 2026-10-15T03:21:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:11.958+00:00 2026-10-15T03:21:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:12.963+00:00 2026-10-15T03:21:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:13.968+00:00 2026-10-15T03:21:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:14.973+00:00 2026-10-15T03:21:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:15.979+00:00 2026-10-15T03:21:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:16.997+00:00 2026-10-15T03:21:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:17.989+00:00 2026-10-15T03:21:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:19.013+00:00 2026-10-15T03:21:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:20.001+00:00 2026-10-15T03:21:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:21.005+00:00 2026-10-15T03:21:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:22.011+00:00 2026-10-15T03:21:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:23.018+00:00 2026-10-15T03:21:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:24.113+00:00 2026-10-15T03:21:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:25.029+00:00 2026-10-15T03:21:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:26.033+00:00 2026-10-15T03:21:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:27.040+00:00 2026-10-15T03:21:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:28.055+00:00 2026-10-15T03:21:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:29.156+00:00 2026-10-15T03:21:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:30.056+00:00 2026-10-15T03:21:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:31.075+00:00 2026-10-15T03:21:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:32.066+00:00 2026-10-15T03:21:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:33.071+00:00 2026-10-15T03:21:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:34.077+00:00 2026-10-15T03:21:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:35.084+00:00 2026-10-15T03:21:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:36.091+00:00 2026-10-15T03:21:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:37.094+00:00 2026-10-15T03:21:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:38.099+00:00 2026-10-15T03:21:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:39.105+00:00 2026-10-15T03:21:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:40.112+00:00 2026-10-15T03:21:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:41.118+00:00 2026-10-15T03:21:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:42.121+00:00 2026-10-15T03:21:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:43.129+00:00 2026-10-15T03:21:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:44.135+00:00 2026-10-15T03:21:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:45.140+00:00 2026-10-15T03:21:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:46.144+00:00 2026-10-15T03:21:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:47.150+00:00 2026-10-15T03:21:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:48.155+00:00 2026-10-15T03:21:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:49.159+00:00 2026-10-15T03:21:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:50.164+00:00 2026-10-15T03:21:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:51.169+00:00 2026-10-15T03:21:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:52.178+00:00 2026-10-15T03:21:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:53.180+00:00 2026-10-15T03:21:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:54.185+00:00 2026-10-15T03:21:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:55.190+00:00 2026-10-15T03:21:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:56.200+00:00 2026-10-15T03:21:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:57.200+00:00 2026-10-15T03:21:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:58.205+00:00 2026-10-15T03:21:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:21:59.209+00:00 2026-10-15T03:21:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:00.214+00:00 2026-10-15T03:22:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:01.249+00:00 2026-10-15T03:22:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:02.224+00:00 2026-10-15T03:22:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:03.232+00:00 2026-10-15T03:22:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:04.242+00:00 2026-10-15T03:22:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:05.248+00:00 2026-10-15T03:22:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:06.251+00:00 2026-10-15T03:22:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:07.251+00:00 2026-10-15T03:22:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:08.255+00:00 2026-10-15T03:22:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:09.260+00:00 2026-10-15T03:22:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:10.270+00:00 2026-10-15T03:22:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:11.276+00:00 2026-10-15T03:22:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:12.278+00:00 2026-10-15T03:22:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:13.283+00:00 2026-10-15T03:22:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:14.289+00:00 2026-10-15T03:22:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:15.294+00:00 2026-10-15T03:22:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:16.298+00:00 2026-10-15T03:22:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:17.304+00:00 2026-10-15T03:22:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:18.309+00:00 2026-10-15T03:22:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:19.314+00:00 2026-10-15T03:22:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:20.321+00:00 2026-10-15T03:22:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:21.325+00:00 2026-10-15T03:22:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:22.332+00:00 2026-10-15T03:22:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:23.337+00:00 2026-10-15T03:22:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:24.341+00:00 2026-10-15T03:22:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:25.349+00:00 2026-10-15T03:22:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:26.352+00:00 2026-10-15T03:22:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:27.358+00:00 2026-10-15T03:22:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:28.362+00:00 2026-10-15T03:22:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:29.368+00:00 2026-10-15T03:22:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:30.468+00:00 2026-10-15T03:22:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:31.380+00:00 2026-10-15T03:22:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:32.385+00:00 2026-10-15T03:22:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:33.391+00:00 2026-10-15T03:22:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:34.396+00:00 2026-10-15T03:22:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:35.400+00:00 2026-10-15T03:22:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:36.406+00:00 2026-10-15T03:22:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:37.410+00:00 2026-10-15T03:22:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:38.422+00:00 2026-10-15T03:22:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:39.421+00:00 2026-10-15T03:22:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:40.427+00:00 2026-10-15T03:22:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:41.431+00:00 2026-10-15T03:22:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:42.435+00:00 2026-10-15T03:22:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:43.441+00:00 2026-10-15T03:22:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:44.445+00:00 2026-10-15T03:22:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:45.481+00:00 2026-10-15T03:22:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:46.457+00:00 2026-10-15T03:22:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:47.461+00:00 2026-10-15T03:22:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:48.467+00:00 2026-10-15T03:22:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:49.471+00:00 2026-10-15T03:22:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:50.477+00:00 2026-10-15T03:22:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:51.483+00:00 2026-10-15T03:22:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:52.488+00:00 2026-10-15T03:22:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:53.492+00:00 2026-10-15T03:22:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:54.499+00:00 2026-10-15T03:22:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:55.503+00:00 2026-10-15T03:22:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:56.507+00:00 2026-10-15T03:22:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:57.512+00:00 2026-10-15T03:22:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:58.517+00:00 2026-10-15T03:22:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:22:59.533+00:00 2026-10-15T03:22:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:00.528+00:00 2026-10-15T03:23:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:01.533+00:00 2026-10-15T03:23:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:02.539+00:00 2026-10-15T03:23:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:03.548+00:00 2026-10-15T03:23:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:04.549+00:00 2026-10-15T03:23:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:05.557+00:00 2026-10-15T03:23:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:06.559+00:00 2026-10-15T03:23:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:07.573+00:00 2026-10-15T03:23:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:08.570+00:00 2026-10-15T03:23:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:09.575+00:00 2026-10-15T03:23:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:10.581+00:00 2026-10-15T03:23:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:11.584+00:00 2026-10-15T03:23:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:12.590+00:00 2026-10-15T03:23:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:13.595+00:00 2026-10-15T03:23:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:14.599+00:00 2026-10-15T03:23:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:15.604+00:00 2026-10-15T03:23:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:16.609+00:00 2026-10-15T03:23:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:17.614+00:00 2026-10-15T03:23:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:18.751+00:00 2026-10-15T03:23:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:19.624+00:00 2026-10-15T03:23:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:20.629+00:00 2026-10-15T03:23:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:21.669+00:00 2026-10-15T03:23:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:22.641+00:00 2026-10-15T03:23:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:23.658+00:00 2026-10-15T03:23:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:24.650+00:00 2026-10-15T03:23:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:25.656+00:00 2026-10-15T03:23:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:26.661+00:00 2026-10-15T03:23:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:27.665+00:00 2026-10-15T03:23:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:28.672+00:00 2026-10-15T03:23:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:29.741+00:00 2026-10-15T03:23:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:30.681+00:00 2026-10-15T03:23:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:31.687+00:00 2026-10-15T03:23:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:32.696+00:00 2026-10-15T03:23:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:33.702+00:00 2026-10-15T03:23:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:34.702+00:00 2026-10-15T03:23:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:35.708+00:00 2026-10-15T03:23:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:36.714+00:00 2026-10-15T03:23:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:37.720+00:00 2026-10-15T03:23:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:38.723+00:00 2026-10-15T03:23:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:39.728+00:00 2026-10-15T03:23:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:40.734+00:00 2026-10-15T03:23:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:41.739+00:00 2026-10-15T03:23:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:42.745+00:00 2026-10-15T03:23:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:43.750+00:00 2026-10-15T03:23:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:44.758+00:00 2026-10-15T03:23:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:45.762+00:00 2026-10-15T03:23:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:46.766+00:00 2026-10-15T03:23:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:47.810+00:00 2026-10-15T03:23:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:48.777+00:00 2026-10-15T03:23:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:49.783+00:00 2026-10-15T03:23:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:50.787+00:00 2026-10-15T03:23:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:51.791+00:00 2026-10-15T03:23:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:52.801+00:00 2026-10-15T03:23:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:53.801+00:00 2026-10-15T03:23:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:54.805+00:00 2026-10-15T03:23:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:55.817+00:00 2026-10-15T03:23:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:56.816+00:00 2026-10-15T03:23:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:57.827+00:00 2026-10-15T03:23:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:58.826+00:00 2026-10-15T03:23:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:23:59.833+00:00 2026-10-15T03:23:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:00.836+00:00 2026-10-15T03:24:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:01.842+00:00 2026-10-15T03:24:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:02.854+00:00 2026-10-15T03:24:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:03.856+00:00 2026-10-15T03:24:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:04.860+00:00 2026-10-15T03:24:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:05.866+00:00 2026-10-15T03:24:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:06.872+00:00 2026-10-15T03:24:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:07.879+00:00 2026-10-15T03:24:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:08.883+00:00 2026-10-15T03:24:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:09.889+00:00 2026-10-15T03:24:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:10.893+00:00 2026-10-15T03:24:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:11.903+00:00 2026-10-15T03:24:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:12.904+00:00 2026-10-15T03:24:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:14.099+00:00 2026-10-15T03:24:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:14.918+00:00 2026-10-15T03:24:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:15.920+00:00 2026-10-15T03:24:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:16.967+00:00 2026-10-15T03:24:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:17.929+00:00 2026-10-15T03:24:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:18.934+00:00 2026-10-15T03:24:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:19.939+00:00 2026-10-15T03:24:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:20.945+00:00 2026-10-15T03:24:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:21.958+00:00 2026-10-15T03:24:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:22.955+00:00 2026-10-15T03:24:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:23.960+00:00 2026-10-15T03:24:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:24.965+00:00 2026-10-15T03:24:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:25.970+00:00 2026-10-15T03:24:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:26.975+00:00 2026-10-15T03:24:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:27.979+00:00 2026-10-15T03:24:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:28.984+00:00 2026-10-15T03:24:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:29.989+00:00 2026-10-15T03:24:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:30.995+00:00 2026-10-15T03:24:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:32.010+00:00 2026-10-15T03:24:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:33.005+00:00 2026-10-15T03:24:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:34.021+00:00 2026-10-15T03:24:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:35.015+00:00 2026-10-15T03:24:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:36.021+00:00 2026-10-15T03:24:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:37.025+00:00 2026-10-15T03:24:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:38.033+00:00 2026-10-15T03:24:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:39.058+00:00 2026-10-15T03:24:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:40.041+00:00 2026-10-15T03:24:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:41.046+00:00 2026-10-15T03:24:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:42.061+00:00 2026-10-15T03:24:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:43.057+00:00 2026-10-15T03:24:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:44.066+00:00 2026-10-15T03:24:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:45.066+00:00 2026-10-15T03:24:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:46.071+00:00 2026-10-15T03:24:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:47.078+00:00 2026-10-15T03:24:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:48.082+00:00 2026-10-15T03:24:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:49.086+00:00 2026-10-15T03:24:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:50.092+00:00 2026-10-15T03:24:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:51.096+00:00 2026-10-15T03:24:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:52.102+00:00 2026-10-15T03:24:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:53.108+00:00 2026-10-15T03:24:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:54.113+00:00 2026-10-15T03:24:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:55.119+00:00 2026-10-15T03:24:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:56.132+00:00 2026-10-15T03:24:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:57.144+00:00 2026-10-15T03:24:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:58.140+00:00 2026-10-15T03:24:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:24:59.145+00:00 2026-10-15T03:24:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:00.150+00:00 2026-10-15T03:25:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:01.158+00:00 2026-10-15T03:25:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:02.159+00:00 2026-10-15T03:25:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:03.255+00:00 2026-10-15T03:25:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:04.170+00:00 2026-10-15T03:25:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:05.175+00:00 2026-10-15T03:25:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:06.339+00:00 2026-10-15T03:25:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:07.185+00:00 2026-10-15T03:25:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:08.194+00:00 2026-10-15T03:25:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:09.195+00:00 2026-10-15T03:25:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:10.214+00:00 2026-10-15T03:25:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:11.205+00:00 2026-10-15T03:25:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:12.211+00:00 2026-10-15T03:25:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:13.216+00:00 2026-10-15T03:25:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:14.220+00:00 2026-10-15T03:25:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:15.229+00:00 2026-10-15T03:25:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:16.231+00:00 2026-10-15T03:25:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:17.238+00:00 2026-10-15T03:25:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:18.241+00:00 2026-10-15T03:25:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:19.252+00:00 2026-10-15T03:25:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:20.252+00:00 2026-10-15T03:25:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:21.257+00:00 2026-10-15T03:25:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:22.262+00:00 2026-10-15T03:25:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:23.266+00:00 2026-10-15T03:25:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:24.271+00:00 2026-10-15T03:25:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:25.373+00:00 2026-10-15T03:25:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:26.282+00:00 2026-10-15T03:25:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:27.297+00:00 2026-10-15T03:25:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:28.292+00:00 2026-10-15T03:25:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:29.298+00:00 2026-10-15T03:25:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:30.302+00:00 2026-10-15T03:25:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:31.306+00:00 2026-10-15T03:25:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:32.311+00:00 2026-10-15T03:25:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:33.318+00:00 2026-10-15T03:25:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:34.322+00:00 2026-10-15T03:25:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:35.331+00:00 2026-10-15T03:25:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:36.331+00:00 2026-10-15T03:25:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:37.336+00:00 2026-10-15T03:25:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:38.341+00:00 2026-10-15T03:25:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:39.345+00:00 2026-10-15T03:25:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:40.350+00:00 2026-10-15T03:25:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:41.355+00:00 2026-10-15T03:25:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:42.360+00:00 2026-10-15T03:25:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:43.365+00:00 2026-10-15T03:25:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:44.370+00:00 2026-10-15T03:25:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:45.381+00:00 2026-10-15T03:25:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:46.379+00:00 2026-10-15T03:25:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:47.384+00:00 2026-10-15T03:25:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:48.389+00:00 2026-10-15T03:25:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:49.393+00:00 2026-10-15T03:25:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:50.398+00:00 2026-10-15T03:25:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:51.402+00:00 2026-10-15T03:25:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:52.407+00:00 2026-10-15T03:25:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:53.411+00:00 2026-10-15T03:25:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:54.415+00:00 2026-10-15T03:25:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:55.421+00:00 2026-10-15T03:25:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:56.426+00:00 2026-10-15T03:25:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:57.431+00:00 2026-10-15T03:25:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:58.436+00:00 2026-10-15T03:25:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:25:59.441+00:00 2026-10-15T03:25:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:00.446+00:00 2026-10-15T03:26:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:01.452+00:00 2026-10-15T03:26:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:02.456+00:00 2026-10-15T03:26:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:03.461+00:00 2026-10-15T03:26:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:04.467+00:00 2026-10-15T03:26:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:05.471+00:00 2026-10-15T03:26:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:06.476+00:00 2026-10-15T03:26:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:07.481+00:00 2026-10-15T03:26:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:08.485+00:00 2026-10-15T03:26:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:09.490+00:00 2026-10-15T03:26:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:10.495+00:00 2026-10-15T03:26:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:11.500+00:00 2026-10-15T03:26:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:12.508+00:00 2026-10-15T03:26:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:13.510+00:00 2026-10-15T03:26:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:14.515+00:00 2026-10-15T03:26:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:15.528+00:00 2026-10-15T03:26:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:16.524+00:00 2026-10-15T03:26:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:17.529+00:00 2026-10-15T03:26:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:18.533+00:00 2026-10-15T03:26:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:19.538+00:00 2026-10-15T03:26:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:20.544+00:00 2026-10-15T03:26:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:21.549+00:00 2026-10-15T03:26:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:22.554+00:00 2026-10-15T03:26:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:23.558+00:00 2026-10-15T03:26:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:24.563+00:00 2026-10-15T03:26:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:25.570+00:00 2026-10-15T03:26:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:26.573+00:00 2026-10-15T03:26:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:27.654+00:00 2026-10-15T03:26:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:28.583+00:00 2026-10-15T03:26:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:29.588+00:00 2026-10-15T03:26:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:30.593+00:00 2026-10-15T03:26:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:31.598+00:00 2026-10-15T03:26:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:32.622+00:00 2026-10-15T03:26:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:33.610+00:00 2026-10-15T03:26:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:34.616+00:00 2026-10-15T03:26:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:35.624+00:00 2026-10-15T03:26:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:36.625+00:00 2026-10-15T03:26:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:37.635+00:00 2026-10-15T03:26:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:38.635+00:00 2026-10-15T03:26:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:39.641+00:00 2026-10-15T03:26:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:40.644+00:00 2026-10-15T03:26:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:41.651+00:00 2026-10-15T03:26:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:42.655+00:00 2026-10-15T03:26:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:43.661+00:00 2026-10-15T03:26:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:44.665+00:00 2026-10-15T03:26:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:45.670+00:00 2026-10-15T03:26:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:46.674+00:00 2026-10-15T03:26:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:47.679+00:00 2026-10-15T03:26:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:48.685+00:00 2026-10-15T03:26:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:49.693+00:00 2026-10-15T03:26:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:50.695+00:00 2026-10-15T03:26:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:51.701+00:00 2026-10-15T03:26:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:52.708+00:00 2026-10-15T03:26:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:53.712+00:00 2026-10-15T03:26:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:54.715+00:00 2026-10-15T03:26:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:55.724+00:00 2026-10-15T03:26:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:56.726+00:00 2026-10-15T03:26:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:57.732+00:00 2026-10-15T03:26:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:58.736+00:00 2026-10-15T03:26:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:26:59.742+00:00 2026-10-15T03:26:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:00.746+00:00 2026-10-15T03:27:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:01.836+00:00 2026-10-15T03:27:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:02.756+00:00 2026-10-15T03:27:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:03.763+00:00 2026-10-15T03:27:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:04.768+00:00 2026-10-15T03:27:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:05.772+00:00 2026-10-15T03:27:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:06.778+00:00 2026-10-15T03:27:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:07.783+00:00 2026-10-15T03:27:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:08.790+00:00 2026-10-15T03:27:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:09.792+00:00 2026-10-15T03:27:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:10.797+00:00 2026-10-15T03:27:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:11.801+00:00 2026-10-15T03:27:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:12.808+00:00 2026-10-15T03:27:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:13.813+00:00 2026-10-15T03:27:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:14.816+00:00 2026-10-15T03:27:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:15.821+00:00 2026-10-15T03:27:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:16.826+00:00 2026-10-15T03:27:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:17.831+00:00 2026-10-15T03:27:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:18.837+00:00 2026-10-15T03:27:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:19.843+00:00 2026-10-15T03:27:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:20.847+00:00 2026-10-15T03:27:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:21.855+00:00 2026-10-15T03:27:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:22.857+00:00 2026-10-15T03:27:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:23.862+00:00 2026-10-15T03:27:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:24.867+00:00 2026-10-15T03:27:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:25.872+00:00 2026-10-15T03:27:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:26.877+00:00 2026-10-15T03:27:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:27.887+00:00 2026-10-15T03:27:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:28.888+00:00 2026-10-15T03:27:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:29.896+00:00 2026-10-15T03:27:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:30.898+00:00 2026-10-15T03:27:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:31.904+00:00 2026-10-15T03:27:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:32.909+00:00 2026-10-15T03:27:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:33.914+00:00 2026-10-15T03:27:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:34.921+00:00 2026-10-15T03:27:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:35.927+00:00 2026-10-15T03:27:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:36.932+00:00 2026-10-15T03:27:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:37.938+00:00 2026-10-15T03:27:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:38.943+00:00 2026-10-15T03:27:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:39.947+00:00 2026-10-15T03:27:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:40.954+00:00 2026-10-15T03:27:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:41.957+00:00 2026-10-15T03:27:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:42.963+00:00 2026-10-15T03:27:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:43.969+00:00 2026-10-15T03:27:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:44.973+00:00 2026-10-15T03:27:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:45.978+00:00 2026-10-15T03:27:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:46.991+00:00 2026-10-15T03:27:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:47.998+00:00 2026-10-15T03:27:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:48.996+00:00 2026-10-15T03:27:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:50.001+00:00 2026-10-15T03:27:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:51.022+00:00 2026-10-15T03:27:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:52.011+00:00 2026-10-15T03:27:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:53.014+00:00 2026-10-15T03:27:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:54.020+00:00 2026-10-15T03:27:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:55.024+00:00 2026-10-15T03:27:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:56.029+00:00 2026-10-15T03:27:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:57.034+00:00 2026-10-15T03:27:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:58.040+00:00 2026-10-15T03:27:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:27:59.180+00:00 2026-10-15T03:27:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:00.060+00:00 2026-10-15T03:28:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:01.137+00:00 2026-10-15T03:28:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:02.060+00:00 2026-10-15T03:28:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:03.064+00:00 2026-10-15T03:28:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:04.069+00:00 2026-10-15T03:28:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:05.074+00:00 2026-10-15T03:28:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:06.148+00:00 2026-10-15T03:28:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:07.084+00:00 2026-10-15T03:28:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:08.089+00:00 2026-10-15T03:28:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:09.094+00:00 2026-10-15T03:28:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:10.101+00:00 2026-10-15T03:28:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:11.264+00:00 2026-10-15T03:28:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:12.115+00:00 2026-10-15T03:28:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:13.118+00:00 2026-10-15T03:28:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:14.124+00:00 2026-10-15T03:28:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:15.129+00:00 2026-10-15T03:28:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:16.135+00:00 2026-10-15T03:28:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:17.140+00:00 2026-10-15T03:28:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:18.145+00:00 2026-10-15T03:28:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:19.150+00:00 2026-10-15T03:28:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:20.162+00:00 2026-10-15T03:28:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:21.163+00:00 2026-10-15T03:28:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:22.166+00:00 2026-10-15T03:28:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:23.170+00:00 2026-10-15T03:28:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:24.176+00:00 2026-10-15T03:28:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:25.181+00:00 2026-10-15T03:28:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:26.187+00:00 2026-10-15T03:28:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:27.196+00:00 2026-10-15T03:28:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:28.197+00:00 2026-10-15T03:28:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:29.202+00:00 2026-10-15T03:28:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:30.209+00:00 2026-10-15T03:28:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:31.214+00:00 2026-10-15T03:28:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:32.221+00:00 2026-10-15T03:28:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:33.225+00:00 2026-10-15T03:28:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:34.230+00:00 2026-10-15T03:28:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:35.235+00:00 2026-10-15T03:28:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:36.242+00:00 2026-10-15T03:28:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:37.248+00:00 2026-10-15T03:28:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:38.253+00:00 2026-10-15T03:28:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:39.264+00:00 2026-10-15T03:28:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:40.264+00:00 2026-10-15T03:28:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:41.270+00:00 2026-10-15T03:28:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:42.274+00:00 2026-10-15T03:28:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:43.418+00:00 2026-10-15T03:28:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:44.284+00:00 2026-10-15T03:28:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:45.289+00:00 2026-10-15T03:28:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:46.294+00:00 2026-10-15T03:28:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:47.300+00:00 2026-10-15T03:28:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:48.304+00:00 2026-10-15T03:28:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:49.309+00:00 2026-10-15T03:28:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:50.314+00:00 2026-10-15T03:28:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:51.321+00:00 2026-10-15T03:28:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:52.325+00:00 2026-10-15T03:28:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:53.330+00:00 2026-10-15T03:28:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:54.334+00:00 2026-10-15T03:28:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:55.340+00:00 2026-10-15T03:28:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:56.345+00:00 2026-10-15T03:28:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:57.349+00:00 2026-10-15T03:28:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:58.354+00:00 2026-10-15T03:28:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:28:59.358+00:00 2026-10-15T03:28:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:00.364+00:00 2026-10-15T03:29:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:01.370+00:00 2026-10-15T03:29:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:02.374+00:00 2026-10-15T03:29:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:03.380+00:00 2026-10-15T03:29:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:04.385+00:00 2026-10-15T03:29:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:05.389+00:00 2026-10-15T03:29:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:06.395+00:00 2026-10-15T03:29:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:07.401+00:00 2026-10-15T03:29:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:08.404+00:00 2026-10-15T03:29:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:09.409+00:00 2026-10-15T03:29:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:10.414+00:00 2026-10-15T03:29:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:11.421+00:00 2026-10-15T03:29:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:12.424+00:00 2026-10-15T03:29:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:13.429+00:00 2026-10-15T03:29:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:14.436+00:00 2026-10-15T03:29:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:15.439+00:00 2026-10-15T03:29:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:16.445+00:00 2026-10-15T03:29:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:17.452+00:00 2026-10-15T03:29:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:18.461+00:00 2026-10-15T03:29:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:19.465+00:00 2026-10-15T03:29:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:20.465+00:00 2026-10-15T03:29:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:21.471+00:00 2026-10-15T03:29:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:22.476+00:00 2026-10-15T03:29:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:23.481+00:00 2026-10-15T03:29:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:24.485+00:00 2026-10-15T03:29:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:25.489+00:00 2026-10-15T03:29:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:26.501+00:00 2026-10-15T03:29:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:27.510+00:00 2026-10-15T03:29:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:28.503+00:00 2026-10-15T03:29:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:29.509+00:00 2026-10-15T03:29:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:30.514+00:00 2026-10-15T03:29:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:31.518+00:00 2026-10-15T03:29:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:32.523+00:00 2026-10-15T03:29:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:33.527+00:00 2026-10-15T03:29:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:34.532+00:00 2026-10-15T03:29:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:35.537+00:00 2026-10-15T03:29:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:36.546+00:00 2026-10-15T03:29:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:37.546+00:00 2026-10-15T03:29:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:38.557+00:00 2026-10-15T03:29:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:39.584+00:00 2026-10-15T03:29:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:40.572+00:00 2026-10-15T03:29:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:41.569+00:00 2026-10-15T03:29:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:42.571+00:00 2026-10-15T03:29:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:43.576+00:00 2026-10-15T03:29:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:44.600+00:00 2026-10-15T03:29:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:45.586+00:00 2026-10-15T03:29:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:46.592+00:00 2026-10-15T03:29:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:47.596+00:00 2026-10-15T03:29:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:48.601+00:00 2026-10-15T03:29:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:49.609+00:00 2026-10-15T03:29:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:50.612+00:00 2026-10-15T03:29:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:51.617+00:00 2026-10-15T03:29:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:52.622+00:00 2026-10-15T03:29:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:53.639+00:00 2026-10-15T03:29:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:54.633+00:00 2026-10-15T03:29:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:55.637+00:00 2026-10-15T03:29:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:56.645+00:00 2026-10-15T03:29:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:57.646+00:00 2026-10-15T03:29:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:58.652+00:00 2026-10-15T03:29:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:29:59.656+00:00 2026-10-15T03:29:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:00.672+00:00 2026-10-15T03:30:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:01.668+00:00 2026-10-15T03:30:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:02.675+00:00 2026-10-15T03:30:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:03.677+00:00 2026-10-15T03:30:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:04.683+00:00 2026-10-15T03:30:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:05.688+00:00 2026-10-15T03:30:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:06.694+00:00 2026-10-15T03:30:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:07.700+00:00 2026-10-15T03:30:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:08.820+00:00 2026-10-15T03:30:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:09.709+00:00 2026-10-15T03:30:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:10.715+00:00 2026-10-15T03:30:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:11.720+00:00 2026-10-15T03:30:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:12.726+00:00 2026-10-15T03:30:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:13.730+00:00 2026-10-15T03:30:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:14.736+00:00 2026-10-15T03:30:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:15.741+00:00 2026-10-15T03:30:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:16.746+00:00 2026-10-15T03:30:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:17.752+00:00 2026-10-15T03:30:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:18.755+00:00 2026-10-15T03:30:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:19.799+00:00 2026-10-15T03:30:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:20.767+00:00 2026-10-15T03:30:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:21.772+00:00 2026-10-15T03:30:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:22.777+00:00 2026-10-15T03:30:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:23.782+00:00 2026-10-15T03:30:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:24.788+00:00 2026-10-15T03:30:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:25.791+00:00 2026-10-15T03:30:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:26.796+00:00 2026-10-15T03:30:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:27.801+00:00 2026-10-15T03:30:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:28.806+00:00 2026-10-15T03:30:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:29.812+00:00 2026-10-15T03:30:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:30.819+00:00 2026-10-15T03:30:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:31.826+00:00 2026-10-15T03:30:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:32.827+00:00 2026-10-15T03:30:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:33.832+00:00 2026-10-15T03:30:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:34.838+00:00 2026-10-15T03:30:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:35.844+00:00 2026-10-15T03:30:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:36.848+00:00 2026-10-15T03:30:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:37.853+00:00 2026-10-15T03:30:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:38.859+00:00 2026-10-15T03:30:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:39.954+00:00 2026-10-15T03:30:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:40.869+00:00 2026-10-15T03:30:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:41.876+00:00 2026-10-15T03:30:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:42.879+00:00 2026-10-15T03:30:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:43.884+00:00 2026-10-15T03:30:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:44.889+00:00 2026-10-15T03:30:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:45.893+00:00 2026-10-15T03:30:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:46.913+00:00 2026-10-15T03:30:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:47.902+00:00 2026-10-15T03:30:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:48.907+00:00 2026-10-15T03:30:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:49.914+00:00 2026-10-15T03:30:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:50.919+00:00 2026-10-15T03:30:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:51.923+00:00 2026-10-15T03:30:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:52.928+00:00 2026-10-15T03:30:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:53.933+00:00 2026-10-15T03:30:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:54.937+00:00 2026-10-15T03:30:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:55.943+00:00 2026-10-15T03:30:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:56.947+00:00 2026-10-15T03:30:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:57.953+00:00 2026-10-15T03:30:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:58.957+00:00 2026-10-15T03:30:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:30:59.964+00:00 2026-10-15T03:30:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:30:59.964+00:00 2026-10-15T03:30:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:00.976+00:00 2026-10-15T03:31:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:00.976+00:00 2026-10-15T03:31:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:01.974+00:00 2026-10-15T03:31:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:01.975+00:00 2026-10-15T03:31:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:02.978+00:00 2026-10-15T03:31:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:02.978+00:00 2026-10-15T03:31:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:03.984+00:00 2026-10-15T03:31:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:03.984+00:00 2026-10-15T03:31:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:04.990+00:00 2026-10-15T03:31:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:04.990+00:00 2026-10-15T03:31:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:05.996+00:00 2026-10-15T03:31:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:05.996+00:00 2026-10-15T03:31:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:07.000+00:00 2026-10-15T03:31:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:07.000+00:00 2026-10-15T03:31:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:08.011+00:00 2026-10-15T03:31:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:08.011+00:00 2026-10-15T03:31:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:09.011+00:00 2026-10-15T03:31:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:09.011+00:00 2026-10-15T03:31:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:10.020+00:00 2026-10-15T03:31:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:10.020+00:00 2026-10-15T03:31:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:11.025+00:00 2026-10-15T03:31:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:11.025+00:00 2026-10-15T03:31:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:12.030+00:00 2026-10-15T03:31:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:12.030+00:00 2026-10-15T03:31:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:13.042+00:00 2026-10-15T03:31:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:13.043+00:00 2026-10-15T03:31:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:14.041+00:00 2026-10-15T03:31:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:14.041+00:00 2026-10-15T03:31:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:15.114+00:00 2026-10-15T03:31:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:15.114+00:00 2026-10-15T03:31:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:16.202+00:00 2026-10-15T03:31:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:16.203+00:00 2026-10-15T03:31:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:17.057+00:00 2026-10-15T03:31:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:17.057+00:00 2026-10-15T03:31:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:18.062+00:00 2026-10-15T03:31:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:18.063+00:00 2026-10-15T03:31:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:19.066+00:00 2026-10-15T03:31:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:19.067+00:00 2026-10-15T03:31:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:20.072+00:00 2026-10-15T03:31:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:20.073+00:00 2026-10-15T03:31:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:21.115+00:00 2026-10-15T03:31:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:21.115+00:00 2026-10-15T03:31:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:22.086+00:00 2026-10-15T03:31:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:22.086+00:00 2026-10-15T03:31:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:23.091+00:00 2026-10-15T03:31:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:23.091+00:00 2026-10-15T03:31:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:24.096+00:00 2026-10-15T03:31:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:24.096+00:00 2026-10-15T03:31:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:25.101+00:00 2026-10-15T03:31:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:25.101+00:00 2026-10-15T03:31:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:26.221+00:00 2026-10-15T03:31:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:26.221+00:00 2026-10-15T03:31:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:27.114+00:00 2026-10-15T03:31:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:27.114+00:00 2026-10-15T03:31:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:28.118+00:00 2026-10-15T03:31:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:28.118+00:00 2026-10-15T03:31:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:29.123+00:00 2026-10-15T03:31:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:29.123+00:00 2026-10-15T03:31:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:30.129+00:00 2026-10-15T03:31:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:30.129+00:00 2026-10-15T03:31:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:31.151+00:00 2026-10-15T03:31:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:31.151+00:00 2026-10-15T03:31:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:32.141+00:00 2026-10-15T03:31:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:32.141+00:00 2026-10-15T03:31:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:33.146+00:00 2026-10-15T03:31:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:33.146+00:00 2026-10-15T03:31:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:34.151+00:00 2026-10-15T03:31:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:34.151+00:00 2026-10-15T03:31:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:35.156+00:00 2026-10-15T03:31:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:35.157+00:00 2026-10-15T03:31:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:36.161+00:00 2026-10-15T03:31:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:36.161+00:00 2026-10-15T03:31:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:37.168+00:00 2026-10-15T03:31:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:37.168+00:00 2026-10-15T03:31:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:38.172+00:00 2026-10-15T03:31:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:38.172+00:00 2026-10-15T03:31:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:39.176+00:00 2026-10-15T03:31:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:39.176+00:00 2026-10-15T03:31:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:40.181+00:00 2026-10-15T03:31:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:40.182+00:00 2026-10-15T03:31:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:41.187+00:00 2026-10-15T03:31:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:41.187+00:00 2026-10-15T03:31:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:42.192+00:00 2026-10-15T03:31:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:42.192+00:00 2026-10-15T03:31:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:43.197+00:00 2026-10-15T03:31:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:43.197+00:00 2026-10-15T03:31:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:44.202+00:00 2026-10-15T03:31:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:44.203+00:00 2026-10-15T03:31:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:45.208+00:00 2026-10-15T03:31:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:45.208+00:00 2026-10-15T03:31:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:46.212+00:00 2026-10-15T03:31:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:46.212+00:00 2026-10-15T03:31:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:47.220+00:00 2026-10-15T03:31:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:47.220+00:00 2026-10-15T03:31:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:48.222+00:00 2026-10-15T03:31:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:48.223+00:00 2026-10-15T03:31:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:49.227+00:00 2026-10-15T03:31:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:49.227+00:00 2026-10-15T03:31:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:50.233+00:00 2026-10-15T03:31:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:50.233+00:00 2026-10-15T03:31:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:51.237+00:00 2026-10-15T03:31:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:51.238+00:00 2026-10-15T03:31:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:52.244+00:00 2026-10-15T03:31:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:52.244+00:00 2026-10-15T03:31:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:53.248+00:00 2026-10-15T03:31:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:53.248+00:00 2026-10-15T03:31:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:54.253+00:00 2026-10-15T03:31:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:54.253+00:00 2026-10-15T03:31:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:55.318+00:00 2026-10-15T03:31:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:55.319+00:00 2026-10-15T03:31:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:56.278+00:00 2026-10-15T03:31:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:56.278+00:00 2026-10-15T03:31:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:57.268+00:00 2026-10-15T03:31:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:57.268+00:00 2026-10-15T03:31:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:58.416+00:00 2026-10-15T03:31:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:58.416+00:00 2026-10-15T03:31:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:31:59.278+00:00 2026-10-15T03:31:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:31:59.278+00:00 2026-10-15T03:31:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:00.283+00:00 2026-10-15T03:32:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:00.283+00:00 2026-10-15T03:32:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:01.289+00:00 2026-10-15T03:32:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:01.291+00:00 2026-10-15T03:32:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:02.296+00:00 2026-10-15T03:32:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:02.296+00:00 2026-10-15T03:32:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:03.300+00:00 2026-10-15T03:32:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:03.300+00:00 2026-10-15T03:32:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:04.306+00:00 2026-10-15T03:32:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:04.306+00:00 2026-10-15T03:32:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:05.315+00:00 2026-10-15T03:32:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:05.315+00:00 2026-10-15T03:32:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:06.320+00:00 2026-10-15T03:32:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:06.320+00:00 2026-10-15T03:32:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:07.320+00:00 2026-10-15T03:32:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:07.320+00:00 2026-10-15T03:32:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:08.325+00:00 2026-10-15T03:32:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:08.325+00:00 2026-10-15T03:32:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:09.330+00:00 2026-10-15T03:32:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:09.330+00:00 2026-10-15T03:32:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:10.341+00:00 2026-10-15T03:32:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:10.341+00:00 2026-10-15T03:32:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:11.340+00:00 2026-10-15T03:32:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:11.340+00:00 2026-10-15T03:32:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:12.345+00:00 2026-10-15T03:32:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:12.345+00:00 2026-10-15T03:32:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:13.351+00:00 2026-10-15T03:32:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:13.351+00:00 2026-10-15T03:32:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:14.356+00:00 2026-10-15T03:32:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:14.356+00:00 2026-10-15T03:32:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:15.360+00:00 2026-10-15T03:32:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:15.361+00:00 2026-10-15T03:32:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:16.365+00:00 2026-10-15T03:32:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:16.365+00:00 2026-10-15T03:32:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:17.370+00:00 2026-10-15T03:32:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:17.370+00:00 2026-10-15T03:32:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:18.376+00:00 2026-10-15T03:32:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:18.376+00:00 2026-10-15T03:32:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:19.381+00:00 2026-10-15T03:32:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:19.381+00:00 2026-10-15T03:32:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:20.394+00:00 2026-10-15T03:32:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:20.394+00:00 2026-10-15T03:32:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:21.392+00:00 2026-10-15T03:32:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:21.392+00:00 2026-10-15T03:32:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:22.396+00:00 2026-10-15T03:32:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:22.396+00:00 2026-10-15T03:32:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:23.400+00:00 2026-10-15T03:32:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:23.400+00:00 2026-10-15T03:32:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:24.406+00:00 2026-10-15T03:32:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:24.406+00:00 2026-10-15T03:32:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:25.410+00:00 2026-10-15T03:32:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:25.411+00:00 2026-10-15T03:32:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:26.415+00:00 2026-10-15T03:32:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:26.415+00:00 2026-10-15T03:32:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:27.423+00:00 2026-10-15T03:32:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:27.423+00:00 2026-10-15T03:32:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:28.424+00:00 2026-10-15T03:32:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:28.424+00:00 2026-10-15T03:32:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:29.431+00:00 2026-10-15T03:32:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:29.431+00:00 2026-10-15T03:32:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:30.436+00:00 2026-10-15T03:32:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:30.436+00:00 2026-10-15T03:32:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:31.441+00:00 2026-10-15T03:32:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:31.441+00:00 2026-10-15T03:32:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:32.446+00:00 2026-10-15T03:32:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:32.446+00:00 2026-10-15T03:32:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:33.452+00:00 2026-10-15T03:32:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:33.452+00:00 2026-10-15T03:32:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:34.607+00:00 2026-10-15T03:32:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:34.607+00:00 2026-10-15T03:32:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:35.469+00:00 2026-10-15T03:32:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:35.469+00:00 2026-10-15T03:32:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:36.467+00:00 2026-10-15T03:32:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:36.467+00:00 2026-10-15T03:32:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:37.472+00:00 2026-10-15T03:32:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:37.472+00:00 2026-10-15T03:32:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:38.477+00:00 2026-10-15T03:32:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:38.477+00:00 2026-10-15T03:32:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:39.482+00:00 2026-10-15T03:32:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:39.483+00:00 2026-10-15T03:32:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:40.499+00:00 2026-10-15T03:32:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:40.499+00:00 2026-10-15T03:32:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:41.495+00:00 2026-10-15T03:32:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:41.495+00:00 2026-10-15T03:32:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:42.498+00:00 2026-10-15T03:32:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:42.498+00:00 2026-10-15T03:32:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:43.511+00:00 2026-10-15T03:32:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:43.511+00:00 2026-10-15T03:32:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:44.529+00:00 2026-10-15T03:32:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:44.529+00:00 2026-10-15T03:32:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:45.567+00:00 2026-10-15T03:32:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:45.567+00:00 2026-10-15T03:32:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:46.519+00:00 2026-10-15T03:32:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:46.520+00:00 2026-10-15T03:32:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:47.527+00:00 2026-10-15T03:32:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:47.528+00:00 2026-10-15T03:32:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:48.529+00:00 2026-10-15T03:32:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:48.529+00:00 2026-10-15T03:32:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:49.535+00:00 2026-10-15T03:32:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:49.535+00:00 2026-10-15T03:32:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:50.542+00:00 2026-10-15T03:32:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:50.543+00:00 2026-10-15T03:32:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:51.546+00:00 2026-10-15T03:32:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:51.546+00:00 2026-10-15T03:32:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:52.557+00:00 2026-10-15T03:32:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:52.557+00:00 2026-10-15T03:32:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:53.557+00:00 2026-10-15T03:32:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:53.557+00:00 2026-10-15T03:32:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:54.562+00:00 2026-10-15T03:32:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:54.562+00:00 2026-10-15T03:32:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:55.568+00:00 2026-10-15T03:32:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:55.568+00:00 2026-10-15T03:32:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:56.572+00:00 2026-10-15T03:32:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:56.572+00:00 2026-10-15T03:32:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:57.579+00:00 2026-10-15T03:32:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:57.579+00:00 2026-10-15T03:32:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:58.582+00:00 2026-10-15T03:32:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:58.582+00:00 2026-10-15T03:32:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:32:59.588+00:00 2026-10-15T03:32:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:32:59.588+00:00 2026-10-15T03:32:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:00.593+00:00 2026-10-15T03:33:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:00.593+00:00 2026-10-15T03:33:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:01.598+00:00 2026-10-15T03:33:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:01.599+00:00 2026-10-15T03:33:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:02.605+00:00 2026-10-15T03:33:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:02.605+00:00 2026-10-15T03:33:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:03.608+00:00 2026-10-15T03:33:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:03.608+00:00 2026-10-15T03:33:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:04.614+00:00 2026-10-15T03:33:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:04.614+00:00 2026-10-15T03:33:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:05.620+00:00 2026-10-15T03:33:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:05.620+00:00 2026-10-15T03:33:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:06.624+00:00 2026-10-15T03:33:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:06.624+00:00 2026-10-15T03:33:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:07.629+00:00 2026-10-15T03:33:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:07.629+00:00 2026-10-15T03:33:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:08.635+00:00 2026-10-15T03:33:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:08.635+00:00 2026-10-15T03:33:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:09.640+00:00 2026-10-15T03:33:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:09.640+00:00 2026-10-15T03:33:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:10.646+00:00 2026-10-15T03:33:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:10.646+00:00 2026-10-15T03:33:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:11.651+00:00 2026-10-15T03:33:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:11.652+00:00 2026-10-15T03:33:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:12.663+00:00 2026-10-15T03:33:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:12.664+00:00 2026-10-15T03:33:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:13.667+00:00 2026-10-15T03:33:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:13.667+00:00 2026-10-15T03:33:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:14.667+00:00 2026-10-15T03:33:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:14.667+00:00 2026-10-15T03:33:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:15.671+00:00 2026-10-15T03:33:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:15.671+00:00 2026-10-15T03:33:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:16.676+00:00 2026-10-15T03:33:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:16.676+00:00 2026-10-15T03:33:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:17.681+00:00 2026-10-15T03:33:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:17.681+00:00 2026-10-15T03:33:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:18.686+00:00 2026-10-15T03:33:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:18.686+00:00 2026-10-15T03:33:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:19.695+00:00 2026-10-15T03:33:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:19.695+00:00 2026-10-15T03:33:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:20.696+00:00 2026-10-15T03:33:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:20.696+00:00 2026-10-15T03:33:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:21.715+00:00 2026-10-15T03:33:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:21.715+00:00 2026-10-15T03:33:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:22.706+00:00 2026-10-15T03:33:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:22.706+00:00 2026-10-15T03:33:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:23.711+00:00 2026-10-15T03:33:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:23.711+00:00 2026-10-15T03:33:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:24.717+00:00 2026-10-15T03:33:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:24.717+00:00 2026-10-15T03:33:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:25.721+00:00 2026-10-15T03:33:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:25.721+00:00 2026-10-15T03:33:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:26.728+00:00 2026-10-15T03:33:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:26.728+00:00 2026-10-15T03:33:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:27.733+00:00 2026-10-15T03:33:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:27.733+00:00 2026-10-15T03:33:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:28.737+00:00 2026-10-15T03:33:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:28.737+00:00 2026-10-15T03:33:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:29.936+00:00 2026-10-15T03:33:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:29.937+00:00 2026-10-15T03:33:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:30.747+00:00 2026-10-15T03:33:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:30.747+00:00 2026-10-15T03:33:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:31.752+00:00 2026-10-15T03:33:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:31.752+00:00 2026-10-15T03:33:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:32.759+00:00 2026-10-15T03:33:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:32.760+00:00 2026-10-15T03:33:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:33.772+00:00 2026-10-15T03:33:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:33.772+00:00 2026-10-15T03:33:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:34.769+00:00 2026-10-15T03:33:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:34.769+00:00 2026-10-15T03:33:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:35.776+00:00 2026-10-15T03:33:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:35.776+00:00 2026-10-15T03:33:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:36.779+00:00 2026-10-15T03:33:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:36.779+00:00 2026-10-15T03:33:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:37.784+00:00 2026-10-15T03:33:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:37.784+00:00 2026-10-15T03:33:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:38.789+00:00 2026-10-15T03:33:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:38.790+00:00 2026-10-15T03:33:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:39.795+00:00 2026-10-15T03:33:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:39.795+00:00 2026-10-15T03:33:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:40.799+00:00 2026-10-15T03:33:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:40.799+00:00 2026-10-15T03:33:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:41.806+00:00 2026-10-15T03:33:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:41.806+00:00 2026-10-15T03:33:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:42.809+00:00 2026-10-15T03:33:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:42.809+00:00 2026-10-15T03:33:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:43.815+00:00 2026-10-15T03:33:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:43.815+00:00 2026-10-15T03:33:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:44.887+00:00 2026-10-15T03:33:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:44.887+00:00 2026-10-15T03:33:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:45.826+00:00 2026-10-15T03:33:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:45.826+00:00 2026-10-15T03:33:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:46.831+00:00 2026-10-15T03:33:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:46.831+00:00 2026-10-15T03:33:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:47.836+00:00 2026-10-15T03:33:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:47.836+00:00 2026-10-15T03:33:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:48.841+00:00 2026-10-15T03:33:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:48.841+00:00 2026-10-15T03:33:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:49.847+00:00 2026-10-15T03:33:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:49.847+00:00 2026-10-15T03:33:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:50.852+00:00 2026-10-15T03:33:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:50.852+00:00 2026-10-15T03:33:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:52.084+00:00 2026-10-15T03:33:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:52.085+00:00 2026-10-15T03:33:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:52.866+00:00 2026-10-15T03:33:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:52.866+00:00 2026-10-15T03:33:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:53.867+00:00 2026-10-15T03:33:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:53.867+00:00 2026-10-15T03:33:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:54.874+00:00 2026-10-15T03:33:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:54.874+00:00 2026-10-15T03:33:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:55.880+00:00 2026-10-15T03:33:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:55.880+00:00 2026-10-15T03:33:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:56.885+00:00 2026-10-15T03:33:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:56.885+00:00 2026-10-15T03:33:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:57.889+00:00 2026-10-15T03:33:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:57.889+00:00 2026-10-15T03:33:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:58.895+00:00 2026-10-15T03:33:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:58.895+00:00 2026-10-15T03:33:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:33:59.899+00:00 2026-10-15T03:33:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:33:59.899+00:00 2026-10-15T03:33:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:00.904+00:00 2026-10-15T03:34:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:00.904+00:00 2026-10-15T03:34:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:02.024+00:00 2026-10-15T03:34:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:02.024+00:00 2026-10-15T03:34:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:02.918+00:00 2026-10-15T03:34:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:02.918+00:00 2026-10-15T03:34:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:03.919+00:00 2026-10-15T03:34:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:03.920+00:00 2026-10-15T03:34:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:04.926+00:00 2026-10-15T03:34:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:04.926+00:00 2026-10-15T03:34:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:05.931+00:00 2026-10-15T03:34:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:05.931+00:00 2026-10-15T03:34:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:06.935+00:00 2026-10-15T03:34:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:06.936+00:00 2026-10-15T03:34:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:07.942+00:00 2026-10-15T03:34:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:07.942+00:00 2026-10-15T03:34:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:08.945+00:00 2026-10-15T03:34:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:08.946+00:00 2026-10-15T03:34:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:09.950+00:00 2026-10-15T03:34:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:09.951+00:00 2026-10-15T03:34:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:10.958+00:00 2026-10-15T03:34:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:10.958+00:00 2026-10-15T03:34:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:11.966+00:00 2026-10-15T03:34:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:11.966+00:00 2026-10-15T03:34:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:12.966+00:00 2026-10-15T03:34:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:12.966+00:00 2026-10-15T03:34:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:13.973+00:00 2026-10-15T03:34:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:13.973+00:00 2026-10-15T03:34:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:14.977+00:00 2026-10-15T03:34:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:14.977+00:00 2026-10-15T03:34:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:15.981+00:00 2026-10-15T03:34:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:15.981+00:00 2026-10-15T03:34:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:16.986+00:00 2026-10-15T03:34:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:16.986+00:00 2026-10-15T03:34:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:17.992+00:00 2026-10-15T03:34:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:17.992+00:00 2026-10-15T03:34:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:18.997+00:00 2026-10-15T03:34:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:18.998+00:00 2026-10-15T03:34:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:20.002+00:00 2026-10-15T03:34:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:20.002+00:00 2026-10-15T03:34:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:21.007+00:00 2026-10-15T03:34:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:21.007+00:00 2026-10-15T03:34:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:22.019+00:00 2026-10-15T03:34:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:22.019+00:00 2026-10-15T03:34:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:23.018+00:00 2026-10-15T03:34:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:23.018+00:00 2026-10-15T03:34:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:24.031+00:00 2026-10-15T03:34:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:24.031+00:00 2026-10-15T03:34:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:25.030+00:00 2026-10-15T03:34:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:25.030+00:00 2026-10-15T03:34:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:26.053+00:00 2026-10-15T03:34:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:26.053+00:00 2026-10-15T03:34:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:27.043+00:00 2026-10-15T03:34:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:27.043+00:00 2026-10-15T03:34:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:28.043+00:00 2026-10-15T03:34:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:28.043+00:00 2026-10-15T03:34:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:29.048+00:00 2026-10-15T03:34:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:29.048+00:00 2026-10-15T03:34:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:30.053+00:00 2026-10-15T03:34:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:30.053+00:00 2026-10-15T03:34:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:31.172+00:00 2026-10-15T03:34:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:31.172+00:00 2026-10-15T03:34:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:32.063+00:00 2026-10-15T03:34:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:32.064+00:00 2026-10-15T03:34:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:33.070+00:00 2026-10-15T03:34:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:33.070+00:00 2026-10-15T03:34:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:34.074+00:00 2026-10-15T03:34:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:34.074+00:00 2026-10-15T03:34:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:35.079+00:00 2026-10-15T03:34:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:35.079+00:00 2026-10-15T03:34:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:36.170+00:00 2026-10-15T03:34:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:36.170+00:00 2026-10-15T03:34:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:37.090+00:00 2026-10-15T03:34:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:37.090+00:00 2026-10-15T03:34:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:38.095+00:00 2026-10-15T03:34:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:38.095+00:00 2026-10-15T03:34:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:39.159+00:00 2026-10-15T03:34:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:39.159+00:00 2026-10-15T03:34:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:40.105+00:00 2026-10-15T03:34:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:40.105+00:00 2026-10-15T03:34:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:41.110+00:00 2026-10-15T03:34:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:41.111+00:00 2026-10-15T03:34:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:42.115+00:00 2026-10-15T03:34:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:42.115+00:00 2026-10-15T03:34:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:43.120+00:00 2026-10-15T03:34:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:43.120+00:00 2026-10-15T03:34:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:44.126+00:00 2026-10-15T03:34:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:44.126+00:00 2026-10-15T03:34:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:45.130+00:00 2026-10-15T03:34:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:45.131+00:00 2026-10-15T03:34:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:46.135+00:00 2026-10-15T03:34:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:46.135+00:00 2026-10-15T03:34:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:47.141+00:00 2026-10-15T03:34:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:47.141+00:00 2026-10-15T03:34:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:48.146+00:00 2026-10-15T03:34:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:48.147+00:00 2026-10-15T03:34:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:49.154+00:00 2026-10-15T03:34:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:49.154+00:00 2026-10-15T03:34:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:50.156+00:00 2026-10-15T03:34:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:50.156+00:00 2026-10-15T03:34:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:51.162+00:00 2026-10-15T03:34:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:51.162+00:00 2026-10-15T03:34:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:52.168+00:00 2026-10-15T03:34:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:52.168+00:00 2026-10-15T03:34:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:53.178+00:00 2026-10-15T03:34:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:53.179+00:00 2026-10-15T03:34:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:54.178+00:00 2026-10-15T03:34:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:54.178+00:00 2026-10-15T03:34:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:55.182+00:00 2026-10-15T03:34:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:55.183+00:00 2026-10-15T03:34:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:56.188+00:00 2026-10-15T03:34:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:56.188+00:00 2026-10-15T03:34:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:57.193+00:00 2026-10-15T03:34:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:57.193+00:00 2026-10-15T03:34:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:58.197+00:00 2026-10-15T03:34:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:58.197+00:00 2026-10-15T03:34:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:34:59.202+00:00 2026-10-15T03:34:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:34:59.203+00:00 2026-10-15T03:34:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:00.208+00:00 2026-10-15T03:35:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:00.208+00:00 2026-10-15T03:35:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:01.212+00:00 2026-10-15T03:35:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:01.213+00:00 2026-10-15T03:35:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:02.218+00:00 2026-10-15T03:35:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:02.218+00:00 2026-10-15T03:35:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:03.222+00:00 2026-10-15T03:35:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:03.222+00:00 2026-10-15T03:35:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:04.227+00:00 2026-10-15T03:35:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:04.228+00:00 2026-10-15T03:35:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:05.232+00:00 2026-10-15T03:35:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:05.233+00:00 2026-10-15T03:35:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:06.238+00:00 2026-10-15T03:35:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:06.238+00:00 2026-10-15T03:35:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:07.243+00:00 2026-10-15T03:35:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:07.243+00:00 2026-10-15T03:35:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:08.248+00:00 2026-10-15T03:35:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:08.248+00:00 2026-10-15T03:35:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:09.253+00:00 2026-10-15T03:35:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:09.253+00:00 2026-10-15T03:35:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:10.259+00:00 2026-10-15T03:35:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:10.259+00:00 2026-10-15T03:35:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:11.268+00:00 2026-10-15T03:35:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:11.268+00:00 2026-10-15T03:35:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:12.269+00:00 2026-10-15T03:35:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:12.269+00:00 2026-10-15T03:35:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:13.277+00:00 2026-10-15T03:35:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:13.277+00:00 2026-10-15T03:35:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:14.281+00:00 2026-10-15T03:35:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:14.281+00:00 2026-10-15T03:35:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:15.287+00:00 2026-10-15T03:35:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:15.287+00:00 2026-10-15T03:35:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:16.433+00:00 2026-10-15T03:35:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:16.433+00:00 2026-10-15T03:35:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:17.298+00:00 2026-10-15T03:35:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:17.298+00:00 2026-10-15T03:35:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:18.302+00:00 2026-10-15T03:35:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:18.302+00:00 2026-10-15T03:35:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:19.313+00:00 2026-10-15T03:35:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:19.314+00:00 2026-10-15T03:35:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:20.311+00:00 2026-10-15T03:35:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:20.311+00:00 2026-10-15T03:35:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:21.319+00:00 2026-10-15T03:35:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:21.319+00:00 2026-10-15T03:35:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:22.322+00:00 2026-10-15T03:35:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:22.323+00:00 2026-10-15T03:35:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:23.329+00:00 2026-10-15T03:35:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:23.329+00:00 2026-10-15T03:35:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:24.333+00:00 2026-10-15T03:35:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:24.333+00:00 2026-10-15T03:35:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:25.338+00:00 2026-10-15T03:35:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:25.338+00:00 2026-10-15T03:35:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:26.343+00:00 2026-10-15T03:35:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:26.343+00:00 2026-10-15T03:35:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:27.348+00:00 2026-10-15T03:35:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:27.348+00:00 2026-10-15T03:35:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:28.353+00:00 2026-10-15T03:35:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:28.353+00:00 2026-10-15T03:35:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:29.359+00:00 2026-10-15T03:35:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:29.359+00:00 2026-10-15T03:35:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:30.367+00:00 2026-10-15T03:35:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:30.367+00:00 2026-10-15T03:35:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:31.368+00:00 2026-10-15T03:35:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:31.368+00:00 2026-10-15T03:35:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:32.373+00:00 2026-10-15T03:35:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:32.373+00:00 2026-10-15T03:35:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:33.378+00:00 2026-10-15T03:35:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:33.378+00:00 2026-10-15T03:35:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:34.383+00:00 2026-10-15T03:35:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:34.384+00:00 2026-10-15T03:35:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:35.389+00:00 2026-10-15T03:35:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:35.389+00:00 2026-10-15T03:35:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:36.394+00:00 2026-10-15T03:35:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:36.394+00:00 2026-10-15T03:35:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:37.399+00:00 2026-10-15T03:35:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:37.399+00:00 2026-10-15T03:35:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:38.404+00:00 2026-10-15T03:35:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:38.404+00:00 2026-10-15T03:35:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:39.408+00:00 2026-10-15T03:35:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:39.408+00:00 2026-10-15T03:35:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:40.416+00:00 2026-10-15T03:35:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:40.417+00:00 2026-10-15T03:35:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:41.424+00:00 2026-10-15T03:35:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:41.424+00:00 2026-10-15T03:35:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:42.423+00:00 2026-10-15T03:35:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:42.423+00:00 2026-10-15T03:35:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:43.428+00:00 2026-10-15T03:35:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:43.428+00:00 2026-10-15T03:35:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:44.433+00:00 2026-10-15T03:35:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:44.433+00:00 2026-10-15T03:35:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:45.439+00:00 2026-10-15T03:35:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:45.439+00:00 2026-10-15T03:35:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:46.443+00:00 2026-10-15T03:35:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:46.443+00:00 2026-10-15T03:35:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:47.457+00:00 2026-10-15T03:35:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:47.457+00:00 2026-10-15T03:35:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:48.453+00:00 2026-10-15T03:35:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:48.453+00:00 2026-10-15T03:35:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:49.461+00:00 2026-10-15T03:35:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:49.461+00:00 2026-10-15T03:35:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:50.464+00:00 2026-10-15T03:35:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:50.464+00:00 2026-10-15T03:35:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:51.471+00:00 2026-10-15T03:35:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:51.471+00:00 2026-10-15T03:35:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:52.477+00:00 2026-10-15T03:35:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:52.477+00:00 2026-10-15T03:35:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:53.480+00:00 2026-10-15T03:35:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:53.480+00:00 2026-10-15T03:35:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:54.484+00:00 2026-10-15T03:35:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:54.484+00:00 2026-10-15T03:35:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:55.490+00:00 2026-10-15T03:35:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:55.490+00:00 2026-10-15T03:35:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:56.494+00:00 2026-10-15T03:35:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:56.495+00:00 2026-10-15T03:35:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:57.500+00:00 2026-10-15T03:35:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:57.500+00:00 2026-10-15T03:35:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:58.506+00:00 2026-10-15T03:35:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:58.506+00:00 2026-10-15T03:35:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:35:59.520+00:00 2026-10-15T03:35:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:35:59.520+00:00 2026-10-15T03:35:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:00.520+00:00 2026-10-15T03:36:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:00.520+00:00 2026-10-15T03:36:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:01.522+00:00 2026-10-15T03:36:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:01.522+00:00 2026-10-15T03:36:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:02.527+00:00 2026-10-15T03:36:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:02.527+00:00 2026-10-15T03:36:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:03.533+00:00 2026-10-15T03:36:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:03.533+00:00 2026-10-15T03:36:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:04.544+00:00 2026-10-15T03:36:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:04.544+00:00 2026-10-15T03:36:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:05.544+00:00 2026-10-15T03:36:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:05.544+00:00 2026-10-15T03:36:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:06.550+00:00 2026-10-15T03:36:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:06.550+00:00 2026-10-15T03:36:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:07.556+00:00 2026-10-15T03:36:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:07.556+00:00 2026-10-15T03:36:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:08.559+00:00 2026-10-15T03:36:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:08.559+00:00 2026-10-15T03:36:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:09.574+00:00 2026-10-15T03:36:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:09.575+00:00 2026-10-15T03:36:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:10.570+00:00 2026-10-15T03:36:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:10.570+00:00 2026-10-15T03:36:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:11.576+00:00 2026-10-15T03:36:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:11.576+00:00 2026-10-15T03:36:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:12.581+00:00 2026-10-15T03:36:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:12.581+00:00 2026-10-15T03:36:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:13.588+00:00 2026-10-15T03:36:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:13.588+00:00 2026-10-15T03:36:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:14.592+00:00 2026-10-15T03:36:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:14.593+00:00 2026-10-15T03:36:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:15.597+00:00 2026-10-15T03:36:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:15.598+00:00 2026-10-15T03:36:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:16.602+00:00 2026-10-15T03:36:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:16.602+00:00 2026-10-15T03:36:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:17.608+00:00 2026-10-15T03:36:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:17.608+00:00 2026-10-15T03:36:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:18.612+00:00 2026-10-15T03:36:18Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:18.612+00:00 2026-10-15T03:36:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:18.612+00:00 2026-10-15T03:36:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:19.617+00:00 2026-10-15T03:36:19Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:19.617+00:00 2026-10-15T03:36:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:19.617+00:00 2026-10-15T03:36:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:20.622+00:00 2026-10-15T03:36:20Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:20.622+00:00 2026-10-15T03:36:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:20.622+00:00 2026-10-15T03:36:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:21.627+00:00 2026-10-15T03:36:21Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:21.627+00:00 2026-10-15T03:36:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:21.627+00:00 2026-10-15T03:36:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:22.631+00:00 2026-10-15T03:36:22Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:22.631+00:00 2026-10-15T03:36:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:22.631+00:00 2026-10-15T03:36:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:23.637+00:00 2026-10-15T03:36:23Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:23.637+00:00 2026-10-15T03:36:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:23.637+00:00 2026-10-15T03:36:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:24.665+00:00 2026-10-15T03:36:24Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:24.665+00:00 2026-10-15T03:36:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:24.665+00:00 2026-10-15T03:36:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:25.648+00:00 2026-10-15T03:36:25Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:25.648+00:00 2026-10-15T03:36:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:25.648+00:00 2026-10-15T03:36:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:26.652+00:00 2026-10-15T03:36:26Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:26.652+00:00 2026-10-15T03:36:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:26.652+00:00 2026-10-15T03:36:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:27.657+00:00 2026-10-15T03:36:27Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:27.657+00:00 2026-10-15T03:36:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:27.657+00:00 2026-10-15T03:36:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:28.663+00:00 2026-10-15T03:36:28Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:28.663+00:00 2026-10-15T03:36:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:28.663+00:00 2026-10-15T03:36:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:29.668+00:00 2026-10-15T03:36:29Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:29.668+00:00 2026-10-15T03:36:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:29.668+00:00 2026-10-15T03:36:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:30.672+00:00 2026-10-15T03:36:30Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:30.672+00:00 2026-10-15T03:36:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:30.672+00:00 2026-10-15T03:36:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:31.676+00:00 2026-10-15T03:36:31Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:31.676+00:00 2026-10-15T03:36:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:31.676+00:00 2026-10-15T03:36:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:32.682+00:00 2026-10-15T03:36:32Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:32.682+00:00 2026-10-15T03:36:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:32.682+00:00 2026-10-15T03:36:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:33.688+00:00 2026-10-15T03:36:33Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:33.688+00:00 2026-10-15T03:36:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:33.688+00:00 2026-10-15T03:36:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:34.702+00:00 2026-10-15T03:36:34Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:34.702+00:00 2026-10-15T03:36:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:34.702+00:00 2026-10-15T03:36:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:35.698+00:00 2026-10-15T03:36:35Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:35.698+00:00 2026-10-15T03:36:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:35.698+00:00 2026-10-15T03:36:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:36.704+00:00 2026-10-15T03:36:36Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:36.704+00:00 2026-10-15T03:36:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:36.704+00:00 2026-10-15T03:36:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:37.709+00:00 2026-10-15T03:36:37Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:37.710+00:00 2026-10-15T03:36:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:37.710+00:00 2026-10-15T03:36:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:38.714+00:00 2026-10-15T03:36:38Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:38.714+00:00 2026-10-15T03:36:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:38.715+00:00 2026-10-15T03:36:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:39.719+00:00 2026-10-15T03:36:39Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:39.719+00:00 2026-10-15T03:36:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:39.719+00:00 2026-10-15T03:36:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:40.724+00:00 2026-10-15T03:36:40Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:40.724+00:00 2026-10-15T03:36:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:40.724+00:00 2026-10-15T03:36:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:41.729+00:00 2026-10-15T03:36:41Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:41.729+00:00 2026-10-15T03:36:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:41.729+00:00 2026-10-15T03:36:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:42.734+00:00 2026-10-15T03:36:42Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:42.734+00:00 2026-10-15T03:36:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:42.734+00:00 2026-10-15T03:36:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:43.740+00:00 2026-10-15T03:36:43Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:43.740+00:00 2026-10-15T03:36:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:43.740+00:00 2026-10-15T03:36:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:44.744+00:00 2026-10-15T03:36:44Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:44.744+00:00 2026-10-15T03:36:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:44.744+00:00 2026-10-15T03:36:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:45.749+00:00 2026-10-15T03:36:45Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:45.749+00:00 2026-10-15T03:36:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:45.749+00:00 2026-10-15T03:36:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:46.753+00:00 2026-10-15T03:36:46Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:46.753+00:00 2026-10-15T03:36:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:46.753+00:00 2026-10-15T03:36:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:47.758+00:00 2026-10-15T03:36:47Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:47.758+00:00 2026-10-15T03:36:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:47.758+00:00 2026-10-15T03:36:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:48.771+00:00 2026-10-15T03:36:48Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:48.772+00:00 2026-10-15T03:36:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:48.772+00:00 2026-10-15T03:36:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:36:49.770+00:00 2026-10-15T03:36:49Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T03:36:49.770+00:00 2026-10-15T03:36:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:36:49.770+00:00 2026-10-15T03:36:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt new file mode 100644 index 00000000..6070ba6b --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt @@ -0,0 +1,2 @@ + bytes: 16171806671 0 0 0 16171746116 0 3948316 0 + READ: 18362 18362 0 2864472 16174096460 285 180396 181025 0 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt new file mode 100644 index 00000000..0f0d837e --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt @@ -0,0 +1,2 @@ + bytes: 16393056335 0 0 0 16392991684 0 4002204 0 + READ: 18796 18796 0 2932176 16395397580 292 204482 205130 0 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log b/runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log new file mode 100644 index 00000000..bd7aeca5 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T03:14:14.700+00:00 2026-10-15T03:14:14.660958634Z {"level":"info","ts":"2026-10-15T03:14:14Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"73e6ab21-a0b9-4211-8b1e-8d228cf38838","claim":"claim-lora-qwen-qwen3-8b-0-trainer-013a69b1","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} +2026-10-15T03:31:10.923+00:00 2026-10-15T03:31:10.911470512Z {"level":"info","ts":"2026-10-15T03:31:10Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"2b07d6ec-2b25-4dba-a67d-607db38e8fea","claim":"claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json b/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json new file mode 100644 index 00000000..c474d169 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json @@ -0,0 +1,275 @@ +{ + "case": "C-8b-h100-cold", + "collected": "2026-10-15T03:38:49.573+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 03:14:13.682000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen3-8B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 03:14:13.683000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 03:31:02.295000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "128ed0fe-59ba-43e7-b0f2-7997322d8233" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 03:31:05.464000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 03:31:08.365000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 03:31:10.461000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 03:31:10.462000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 03:31:12.490000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 03:31:12.490000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 03:36:43.884000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4\n\nQuestion: What is " + }, + "client.end": { + "ts": "2026-10-15 03:36:43.886000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen3-8b-0-sampler", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "pod": "orw-lora-qwen-qwen3-8b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 03:31:10+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 03:31:10.295000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 03:31:11.907000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 03:31:11.907000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 03:32:51+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 03:31:11+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 03:31:11+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 03:32:51+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 03:32:51+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 03:31:11+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 03:32:50+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 03:31:11+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "sampler.event_Pulling": { + "ts": "2026-10-15 03:31:15+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 03:32:50+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m35.306s (1m35.306s i", + "kubelet_pull_s": 95.306, + "pulled_from_cache": false + }, + "sampler.event_Created": { + "ts": "2026-10-15 03:32:50+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 03:32:50+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 03:32:57.768739+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 03:33:16.117730+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:33:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 03:33:17.209275+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:33:17 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 03:33:18.247405+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:33:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 03:35:00.792211+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:00 [default_loader.py:430] Loading weights took 102.55 seconds", + "self_reported_s": 102.55 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 03:35:01.444736+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:01 [model_runner.py:428] Model loading took 18.17 GiB memory and 104.493682 second", + "self_reported_s": 104.493682 + }, + "sampler.compile_done": { + "ts": "2026-10-15 03:35:37.706264+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:37 [monitor.py:53] torch.compile took 35.25 s in total", + "self_reported_s": 35.25 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 03:35:57.941422+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:57 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", + "self_reported_s": 16.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 03:35:58.562053+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:58 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 03:36:17.004554+00:00", + "source": "container log: (EngineCore pid=168) INFO 10-15 03:36:17 [core.py:372] init engine (profile, create kv cache, warmup model) took 75.55 s", + "self_reported_s": 75.55 + }, + "sampler.ready_key": { + "ts": "2026-10-15 03:36:18+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" + }, + "trainer": { + "workload": "lora-qwen-qwen3-8b-0-trainer", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "pod": "orw-lora-qwen-qwen3-8b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 03:14:14+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 03:14:14.472000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 03:14:14.472000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 03:14:14.472000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 03:15:52+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 03:14:14+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 03:14:14+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 03:15:52+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 03:15:52+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 03:14:14+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 03:15:51+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 03:14:14+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "trainer.event_Pulling": { + "ts": "2026-10-15 03:14:19+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 03:15:51+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m32.497s (1m32.497s i", + "kubelet_pull_s": 92.497, + "pulled_from_cache": false + }, + "trainer.event_Created": { + "ts": "2026-10-15 03:15:51+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 03:15:51+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 03:15:56.012903+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 03:15:56.267417+00:00", + "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 03:30:59.760141+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 03:30:59.761631+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 03:30:59+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.md b/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.md new file mode 100644 index 00000000..5e9f23b5 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.md @@ -0,0 +1,111 @@ +# Cold start timeline: C-8b-h100-cold + +t=0 is the client's `create_model` call at 2026-10-15T03:14:13.683+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen3-8b-0-trainer` on `gke-open-rl-dra-h100-trainer-702095b4-1g3a` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.workload_created | 03:14:14.000 | 0.3 | | Workload creationTimestamp (apiserver) | +| trainer.pod_created | 03:14:14.000 | 0.3 | | pod creationTimestamp (apiserver) | +| trainer.pod_Initialized | 03:14:14.000 | 0.3 | | pod condition Initialized | +| trainer.pod_PodScheduled | 03:14:14.000 | 0.3 | | pod condition PodScheduled | +| trainer.event_Scheduled | 03:14:14.000 | 0.3 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a | +| trainer.workload_first_seen_by_poller | 03:14:14.472 | 0.8 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 03:14:14.472 | 0.8 | | workloads.tsv poller: status.podName set | +| trainer.workload_reports_node | 03:14:14.472 | 0.8 | | workloads.tsv poller: status.nodeName set | +| trainer.event_Pulling | 03:14:19.000 | 5.3 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" | +| trainer.container_started | 03:15:51.000 | 97.3 | | containerStatuses.state.running.startedAt | +| trainer.event_Pulled | 03:15:51.000 | 97.3 | 92.497 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" in 1m32.497s (1m32.497s i | +| trainer.event_Created | 03:15:51.000 | 97.3 | | event: Container created | +| trainer.event_Started | 03:15:51.000 | 97.3 | | event: Container started | +| trainer.condition_Placed_True | 03:15:52.000 | 98.3 | | Workload condition Placed | +| trainer.pod_Ready | 03:15:52.000 | 98.3 | | pod condition Ready | +| trainer.pod_ContainersReady | 03:15:52.000 | 98.3 | | pod condition ContainersReady | +| trainer.process_first_log | 03:15:56.012 | 102.3 | | container log, first line | +| trainer.load_start | 03:15:56.267 | 102.6 | | container log: Loading base model Qwen/Qwen3-8B to cuda:0... | +| trainer.ready_key | 03:30:59.000 | 1005.3 | | redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B | +| trainer.load_done | 03:30:59.760 | 1006.1 | | container log: Successfully loaded. | +| trainer.processor_started | 03:30:59.761 | 1006.1 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1). | + +## sampler: `orw-lora-qwen-qwen3-8b-0-sampler` on `gke-open-rl-dra-h100-sampler-a721f127-0853` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_created | 03:31:10.000 | 1016.3 | | Workload creationTimestamp (apiserver) | +| sampler.workload_first_seen_by_poller | 03:31:10.295 | 1016.6 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.pod_created | 03:31:11.000 | 1017.3 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 03:31:11.000 | 1017.3 | | pod condition Initialized | +| sampler.pod_PodScheduled | 03:31:11.000 | 1017.3 | | pod condition PodScheduled | +| sampler.event_Scheduled | 03:31:11.000 | 1017.3 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853 | +| sampler.workload_reports_pod | 03:31:11.907 | 1018.2 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 03:31:11.907 | 1018.2 | | workloads.tsv poller: status.nodeName set | +| sampler.event_Pulling | 03:31:15.000 | 1021.3 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" | +| sampler.container_started | 03:32:50.000 | 1116.3 | | containerStatuses.state.running.startedAt | +| sampler.event_Pulled | 03:32:50.000 | 1116.3 | 95.306 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" in 1m35.306s (1m35.306s i | +| sampler.event_Created | 03:32:50.000 | 1116.3 | | event: Container created | +| sampler.event_Started | 03:32:50.000 | 1116.3 | | event: Container started | +| sampler.condition_Placed_True | 03:32:51.000 | 1117.3 | | Workload condition Placed | +| sampler.pod_Ready | 03:32:51.000 | 1117.3 | | pod condition Ready | +| sampler.pod_ContainersReady | 03:32:51.000 | 1117.3 | | pod condition ContainersReady | +| sampler.process_first_log | 03:32:57.768 | 1124.1 | | container log, first line | +| sampler.engine_init_start | 03:33:16.117 | 1142.4 | | container log: (EngineCore pid=168) INFO 10-15 03:33:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 03:33:17.209 | 1143.5 | | container log: (EngineCore pid=168) INFO 10-15 03:33:17 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 03:33:18.247 | 1144.6 | | container log: (EngineCore pid=168) INFO 10-15 03:33:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 03:35:00.792 | 1247.1 | 102.55 | container log: (EngineCore pid=168) INFO 10-15 03:35:00 [default_loader.py:430] Loading weights took 102.55 seconds | +| sampler.model_loaded | 03:35:01.444 | 1247.8 | 104.493682 | container log: (EngineCore pid=168) INFO 10-15 03:35:01 [model_runner.py:428] Model loading took 18.17 GiB memory and 104.493682 second | +| sampler.compile_done | 03:35:37.706 | 1284.0 | 35.25 | container log: (EngineCore pid=168) INFO 10-15 03:35:37 [monitor.py:53] torch.compile took 35.25 s in total | +| sampler.graphs_captured | 03:35:57.941 | 1304.3 | 16.0 | container log: (EngineCore pid=168) INFO 10-15 03:35:57 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB | +| sampler.kv_cache_sized | 03:35:58.562 | 1304.9 | | container log: (EngineCore pid=168) INFO 10-15 03:35:58 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB | +| sampler.engine_init_done | 03:36:17.004 | 1323.3 | 75.55 | container log: (EngineCore pid=168) INFO 10-15 03:36:17 [core.py:372] init engine (profile, create kv cache, warmup model) took 75.55 s | +| sampler.ready_key | 03:36:18.000 | 1324.3 | | redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 03:14:13.682 | -0.0 | +| client.create_model.start | 03:14:13.683 | 0.0 | +| client.create_model.done | 03:31:02.295 | 1008.6 | +| client.forward_backward.start | 03:31:05.464 | 1011.8 | +| client.forward_backward.done | 03:31:08.365 | 1014.7 | +| client.optim_step.done | 03:31:10.461 | 1016.8 | +| client.save_weights_and_get_sampling_client.start | 03:31:10.462 | 1016.8 | +| client.save_weights_and_get_sampling_client.done | 03:31:12.490 | 1018.8 | +| client.sample.start | 03:31:12.490 | 1018.8 | +| client.sample.done | 03:36:43.884 | 1350.2 | +| client.end | 03:36:43.886 | 1350.2 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.3 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 97.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.3 | +| weights load | trainer.load_start | trainer.load_done | 903.5 | +| ready key | trainer.load_done | trainer.ready_key | -0.8 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 1008.6 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.5 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 99.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 7.8 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 18.3 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 103.6 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 36.9 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.3 | +| ready key | sampler.engine_init_done | sampler.ready_key | 1.0 | +| client: first sample blocked | client.sample.start | client.sample.done | 331.4 | + diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt new file mode 100644 index 00000000..433537d3 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt @@ -0,0 +1,14 @@ +device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 + opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none + age: 403 + events: 58 1574 0 0 44 61 1744 0 0 12462 0 0 32 0 28 0 0 23 0 0 0 0 0 0 0 0 0 + bytes: 11495181 0 0 0 3265001325 0 797317 0 + READ: 12459 12459 0 1943604 3266596084 221 584552 585076 0 +device /dev/md0 mounted on /etc/hosts with fstype ext4 +device /dev/md0 mounted on /dev/termination-log with fstype ext4 +device /dev/md0 mounted on /etc/hostname with fstype ext4 +device /dev/md0 mounted on /etc/resolv.conf with fstype ext4 +device shm mounted on /dev/shm with fstype tmpfs +device /dev/nvme0n1p1 mounted on /usr/bin/nvidia-cuda-mps-control with fstype ext4 +device /dev/nvme0n1p1 mounted on /usr/bin/nvidia-cuda-mps-server with fstype ext4 +device /dev/nvme0n1p1 mounted on /usr/bin/nvidia-debugdump with fstype ext4 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt new file mode 100644 index 00000000..b84369a0 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt @@ -0,0 +1,2 @@ + bytes: 11495181 0 0 0 3943458669 0 962892 0 + READ: 15070 15070 0 2350920 3945387636 265 704690 705322 0 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt new file mode 100644 index 00000000..c83d56c7 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T03:14:09.568+00:00 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt new file mode 100644 index 00000000..0d162c07 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json b/runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json new file mode 100644 index 00000000..11c30084 --- /dev/null +++ b/runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T03:31:10Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035171447887008", + "uid": "4637c9e3-a819-432d-9edb-58a1522564c4" + }, + "spec": { + "accelerator": { + "memory": "29Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "name": "worker", + "resources": { + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "464502aa-adbd-4913-8de9-9cf72fa2e24d", + "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T03:32:51Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "29Gi", + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T03:14:14Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792034152699023004", + "uid": "013a69b1-c398-4e81-a5c5-90edf5da837a" + }, + "spec": { + "accelerator": { + "memory": "20Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "name": "worker", + "resources": { + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "bbe9937b-9591-4fe5-9cd5-1845592a917d", + "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T03:15:52Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "20Gi", + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt new file mode 100644 index 00000000..5d1863d9 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt @@ -0,0 +1 @@ +interrupted: port-forward outage ~03:43-04:23 UTC; reaper released workers 03:45:24; trainer load 03:39:13->03:43:39 (266 s) is valid diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log new file mode 100644 index 00000000..246ca3b1 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log @@ -0,0 +1,38 @@ +2026-10-15T03:39:03.941+00:00 2026-10-15T03:39:03.931120235Z INFO: 127.0.0.1:42054 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T03:39:03.975+00:00 2026-10-15T03:39:03.964551019Z INFO: 10.32.3.1:38928 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:39:04.179+00:00 2026-10-15T03:39:04.169671310Z INFO: 127.0.0.1:42060 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T03:39:04.245+00:00 2026-10-15T03:39:04.236451920Z INFO: 127.0.0.1:42060 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T03:39:04.488+00:00 2026-10-15T03:39:04.478554501Z [API_SERVER] enqueued op=create_model request_id=ddb4615c-4bb0-44b0-92a1-c79e668c7324 model_id=ddb4615c-4bb0-44b0-92a1-c79e668c7324 active_set=Qwen/Qwen3-8B-1 +2026-10-15T03:39:04.489+00:00 2026-10-15T03:39:04.479576649Z INFO: 127.0.0.1:42062 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T03:39:13.977+00:00 2026-10-15T03:39:13.964648602Z INFO: 10.32.3.1:37426 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:39:14.434+00:00 2026-10-15T03:39:14.425265125Z INFO: 127.0.0.1:55092 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:39:23.976+00:00 2026-10-15T03:39:23.964483470Z INFO: 10.32.3.1:37262 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:39:33.977+00:00 2026-10-15T03:39:33.964871846Z INFO: 10.32.3.1:58992 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:39:43.975+00:00 2026-10-15T03:39:43.964576360Z INFO: 10.32.3.1:52666 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:39:53.976+00:00 2026-10-15T03:39:53.964501809Z INFO: 10.32.3.1:52688 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:39:54.635+00:00 2026-10-15T03:39:54.609937412Z INFO: 127.0.0.1:51376 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:40:04.035+00:00 2026-10-15T03:40:03.964991692Z INFO: 10.32.3.1:41216 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:40:13.975+00:00 2026-10-15T03:40:13.963993568Z INFO: 10.32.3.1:53146 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:40:23.977+00:00 2026-10-15T03:40:23.965037340Z INFO: 10.32.3.1:40128 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:40:33.977+00:00 2026-10-15T03:40:33.965609459Z INFO: 10.32.3.1:54704 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:40:43.976+00:00 2026-10-15T03:40:43.965003505Z INFO: 10.32.3.1:40814 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:40:44.810+00:00 2026-10-15T03:40:44.800601117Z INFO: 127.0.0.1:59960 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:40:53.976+00:00 2026-10-15T03:40:53.964698725Z INFO: 10.32.3.1:36634 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:41:03.977+00:00 2026-10-15T03:41:03.964982471Z INFO: 10.32.3.1:54336 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:41:13.977+00:00 2026-10-15T03:41:13.964439663Z INFO: 10.32.3.1:57996 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:41:23.977+00:00 2026-10-15T03:41:23.964617258Z INFO: 10.32.3.1:33020 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:41:25.003+00:00 2026-10-15T03:41:24.994100729Z INFO: 127.0.0.1:60116 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:41:33.977+00:00 2026-10-15T03:41:33.964563446Z INFO: 10.32.3.1:39176 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:41:43.978+00:00 2026-10-15T03:41:43.965156711Z INFO: 10.32.3.1:43750 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:41:53.982+00:00 2026-10-15T03:41:53.964453599Z INFO: 10.32.3.1:43488 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:42:03.976+00:00 2026-10-15T03:42:03.965028296Z INFO: 10.32.3.1:56500 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:42:13.976+00:00 2026-10-15T03:42:13.964101051Z INFO: 10.32.3.1:32798 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:42:15.186+00:00 2026-10-15T03:42:15.177104960Z INFO: 127.0.0.1:42622 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:42:23.976+00:00 2026-10-15T03:42:23.964020928Z INFO: 10.32.3.1:56208 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:42:33.976+00:00 2026-10-15T03:42:33.964949522Z INFO: 10.32.3.1:49210 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:42:43.975+00:00 2026-10-15T03:42:43.964068045Z INFO: 10.32.3.1:54612 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:42:53.977+00:00 2026-10-15T03:42:53.964413969Z INFO: 10.32.3.1:43698 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:43:03.976+00:00 2026-10-15T03:43:03.964619748Z INFO: 10.32.3.1:44292 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:43:05.369+00:00 2026-10-15T03:43:05.359540334Z INFO: 127.0.0.1:46534 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T03:43:13.978+00:00 2026-10-15T03:43:13.964574300Z INFO: 10.32.3.1:53490 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T03:48:57.227+00:00 error: http2: client connection lost diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl new file mode 100644 index 00000000..501cd45a --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl @@ -0,0 +1,4 @@ +{"ts": "2026-10-15T03:39:03.686+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T03:39:03.686+00:00", "t": 0.001, "phase": "create_model.start"} +{"ts": "2026-10-15T03:48:18.935+00:00", "t": 555.249, "phase": "create_model.done", "model_id": "ddb4615c-4bb0-44b0-92a1-c79e668c7324"} +{"ts": "2026-10-15T03:48:21.731+00:00", "t": 558.046, "phase": "forward_backward.start"} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log new file mode 100644 index 00000000..a12eda14 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log @@ -0,0 +1,6 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.001s create_model.start +[cold-start] 555.249s create_model.done {'model_id': 'ddb4615c-4bb0-44b0-92a1-c79e668c7324'} +[cold-start] 558.046s forward_backward.start +Retrying request self.request_id='fc48f953-c5c6-4169-b997-c905ba50b668' because of try_again +EXIT=143 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt new file mode 100644 index 00000000..a3312864 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T04:27:09.940+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json new file mode 100644 index 00000000..d1cfc5a9 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json @@ -0,0 +1,401 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:31:11Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071080495023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:31:11Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T03:31:11Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96671460e727", + "namespace": "openrl-system", + "resourceVersion": "1792035071245135014", + "uid": "8acbbf2e-49f9-4202-949a-77c580789a9a" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:31:15Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:31:15Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", + "metadata": { + "creationTimestamp": "2026-10-15T03:31:15Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9668063dcb02", + "namespace": "openrl-system", + "resourceVersion": "1792035075278831019", + "uid": "6912eb66-4b94-4b18-a74e-ab327aeddb23" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:32:50Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:32:50Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m35.306s (1m35.306s including waiting). Image size: 7123551739 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T03:32:50Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e36f0cbc2", + "namespace": "openrl-system", + "resourceVersion": "1792035170612415015", + "uid": "8591c4e1-a15e-4174-ad4a-fd97a845ccf1" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:32:50Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:32:50Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T03:32:50Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3778892c", + "namespace": "openrl-system", + "resourceVersion": "1792035170630543005", + "uid": "ffa2ae1f-d67a-48f5-bbae-e60a320c7ca4" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:32:50Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:32:50Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T03:32:50Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3f40302b", + "namespace": "openrl-system", + "resourceVersion": "1792035170729087005", + "uid": "1519861d-65be-4db6-b777-bd8220ced5f8" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:38:15Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:38:15Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:38:15Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96c9e6a9a8f5", + "namespace": "openrl-system", + "resourceVersion": "1792035495672207024", + "uid": "ddca643d-0c58-4430-9459-5686015bcf61" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:38:15Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792034054934527006", + "uid": "5f7199d4-14aa-4747-9728-d940f5af679f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:38:15Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:38:15Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96c9ee8c225c", + "namespace": "openrl-system", + "resourceVersion": "1792035495812399014", + "uid": "ea4fdc32-2c46-41b7-8537-f60b2a3e7ce3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:04Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544745343006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:04Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:04Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d55baf7ea2", + "namespace": "openrl-system", + "resourceVersion": "1792035544868287013", + "uid": "717619c9-bc6f-4410-89cb-933289a9091a" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:08Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:08Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:08Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d6291f5cc5", + "namespace": "openrl-system", + "resourceVersion": "1792035548309503024", + "uid": "0ad1918b-487b-45d0-95f6-90606fb0f996" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:08Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:08Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:08Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d629aa8317", + "namespace": "openrl-system", + "resourceVersion": "1792035548331599007", + "uid": "a146313b-77f3-450f-9b9d-6d0b1f04c6cf" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:08Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:08Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:08Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d62f39a7f5", + "namespace": "openrl-system", + "resourceVersion": "1792035548412447000", + "uid": "fcf01033-06e2-4cbd-a2c7-51bf347ddf42" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:45:24Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:45:24Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:45:24Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de972dcf359e7a", + "namespace": "openrl-system", + "resourceVersion": "1792035924784175002", + "uid": "b8207d84-11ad-4a25-867c-7be9b78ba580" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json new file mode 100644 index 00000000..f940f10c --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json @@ -0,0 +1,50 @@ +{ + "gke-open-rl-dra-default-pool-095bf129-0flt": [], + "gke-open-rl-dra-default-pool-c19e0101-2bk0": [ + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83" + ], + "gke-open-rl-dra-default-pool-ec729728-6mpx": [], + "gke-open-rl-dra-h100-sampler-a721f127-0853": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-h100-sampler-a721f127-ed3t": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-client:pr271-210aee9", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-h100-trainer-702095b4-1g3a": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" + ], + "gke-open-rl-dra-h100-trainer-702095b4-tuqu": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6" + ], + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" + ] +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json new file mode 100644 index 00000000..f940f10c --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json @@ -0,0 +1,50 @@ +{ + "gke-open-rl-dra-default-pool-095bf129-0flt": [], + "gke-open-rl-dra-default-pool-c19e0101-2bk0": [ + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83" + ], + "gke-open-rl-dra-default-pool-ec729728-6mpx": [], + "gke-open-rl-dra-h100-sampler-a721f127-0853": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-h100-sampler-a721f127-ed3t": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-client:pr271-210aee9", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-h100-trainer-702095b4-1g3a": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" + ], + "gke-open-rl-dra-h100-trainer-702095b4-tuqu": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6" + ], + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" + ], + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8": [ + "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" + ] +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt new file mode 100644 index 00000000..d3c42940 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt @@ -0,0 +1,8 @@ +gke-open-rl-dra-h100-sampler-a721f127-0853 Ready 9d v1.35.7-gke.1222000 true +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 25h v1.35.8-gke.1380001 true +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 25h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json new file mode 100644 index 00000000..5bfc412e --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json @@ -0,0 +1,5980 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-api-server-645ff9777d-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "645ff9777d", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + } + ], + "resourceVersion": "1791941173996783021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-79xdl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:02Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:26:02Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.10", + "podIPs": [ + { + "ip": "10.32.3.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log new file mode 100644 index 00000000..74402189 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log @@ -0,0 +1,1001 @@ +2026-10-15T03:39:00.557+00:00 2026-10-15T03:39:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:00.558+00:00 2026-10-15T03:39:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:01.562+00:00 2026-10-15T03:39:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:01.562+00:00 2026-10-15T03:39:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:02.566+00:00 2026-10-15T03:39:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:02.566+00:00 2026-10-15T03:39:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:03.572+00:00 2026-10-15T03:39:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:03.572+00:00 2026-10-15T03:39:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:04.576+00:00 2026-10-15T03:39:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:04.576+00:00 2026-10-15T03:39:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:05.582+00:00 2026-10-15T03:39:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:05.582+00:00 2026-10-15T03:39:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:06.587+00:00 2026-10-15T03:39:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:06.587+00:00 2026-10-15T03:39:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:07.595+00:00 2026-10-15T03:39:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:07.596+00:00 2026-10-15T03:39:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:08.597+00:00 2026-10-15T03:39:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:08.597+00:00 2026-10-15T03:39:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:09.603+00:00 2026-10-15T03:39:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:09.603+00:00 2026-10-15T03:39:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:10.608+00:00 2026-10-15T03:39:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:10.608+00:00 2026-10-15T03:39:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:11.613+00:00 2026-10-15T03:39:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:11.613+00:00 2026-10-15T03:39:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:12.619+00:00 2026-10-15T03:39:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:12.619+00:00 2026-10-15T03:39:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:13.625+00:00 2026-10-15T03:39:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:13.625+00:00 2026-10-15T03:39:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:14.630+00:00 2026-10-15T03:39:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:14.630+00:00 2026-10-15T03:39:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:15.634+00:00 2026-10-15T03:39:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:15.634+00:00 2026-10-15T03:39:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:16.638+00:00 2026-10-15T03:39:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:16.639+00:00 2026-10-15T03:39:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:17.645+00:00 2026-10-15T03:39:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:17.645+00:00 2026-10-15T03:39:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:18.648+00:00 2026-10-15T03:39:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:18.648+00:00 2026-10-15T03:39:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:19.654+00:00 2026-10-15T03:39:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:19.655+00:00 2026-10-15T03:39:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:20.659+00:00 2026-10-15T03:39:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:20.659+00:00 2026-10-15T03:39:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:21.664+00:00 2026-10-15T03:39:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:21.664+00:00 2026-10-15T03:39:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:22.671+00:00 2026-10-15T03:39:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:22.672+00:00 2026-10-15T03:39:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:23.673+00:00 2026-10-15T03:39:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:23.673+00:00 2026-10-15T03:39:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:24.678+00:00 2026-10-15T03:39:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:24.678+00:00 2026-10-15T03:39:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:25.684+00:00 2026-10-15T03:39:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:25.684+00:00 2026-10-15T03:39:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:26.689+00:00 2026-10-15T03:39:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:26.689+00:00 2026-10-15T03:39:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:27.692+00:00 2026-10-15T03:39:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:27.692+00:00 2026-10-15T03:39:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:28.698+00:00 2026-10-15T03:39:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:28.699+00:00 2026-10-15T03:39:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:29.702+00:00 2026-10-15T03:39:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:29.702+00:00 2026-10-15T03:39:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:30.707+00:00 2026-10-15T03:39:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:30.707+00:00 2026-10-15T03:39:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:31.714+00:00 2026-10-15T03:39:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:31.714+00:00 2026-10-15T03:39:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:32.717+00:00 2026-10-15T03:39:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:32.717+00:00 2026-10-15T03:39:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:33.722+00:00 2026-10-15T03:39:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:33.722+00:00 2026-10-15T03:39:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:34.726+00:00 2026-10-15T03:39:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:34.726+00:00 2026-10-15T03:39:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:35.731+00:00 2026-10-15T03:39:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:35.731+00:00 2026-10-15T03:39:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:36.749+00:00 2026-10-15T03:39:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:36.749+00:00 2026-10-15T03:39:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:37.742+00:00 2026-10-15T03:39:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:37.742+00:00 2026-10-15T03:39:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:38.746+00:00 2026-10-15T03:39:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:38.746+00:00 2026-10-15T03:39:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:39.751+00:00 2026-10-15T03:39:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:39.751+00:00 2026-10-15T03:39:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:40.756+00:00 2026-10-15T03:39:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:40.756+00:00 2026-10-15T03:39:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:41.761+00:00 2026-10-15T03:39:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:41.762+00:00 2026-10-15T03:39:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:42.777+00:00 2026-10-15T03:39:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:42.777+00:00 2026-10-15T03:39:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:43.772+00:00 2026-10-15T03:39:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:43.772+00:00 2026-10-15T03:39:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:44.776+00:00 2026-10-15T03:39:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:44.777+00:00 2026-10-15T03:39:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:45.785+00:00 2026-10-15T03:39:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:45.785+00:00 2026-10-15T03:39:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:46.786+00:00 2026-10-15T03:39:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:46.786+00:00 2026-10-15T03:39:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:47.791+00:00 2026-10-15T03:39:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:47.791+00:00 2026-10-15T03:39:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:48.796+00:00 2026-10-15T03:39:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:48.796+00:00 2026-10-15T03:39:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:49.801+00:00 2026-10-15T03:39:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:49.801+00:00 2026-10-15T03:39:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:50.815+00:00 2026-10-15T03:39:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:50.815+00:00 2026-10-15T03:39:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:51.900+00:00 2026-10-15T03:39:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:51.900+00:00 2026-10-15T03:39:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:52.816+00:00 2026-10-15T03:39:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:52.816+00:00 2026-10-15T03:39:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:53.820+00:00 2026-10-15T03:39:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:53.820+00:00 2026-10-15T03:39:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:54.826+00:00 2026-10-15T03:39:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:54.827+00:00 2026-10-15T03:39:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:55.831+00:00 2026-10-15T03:39:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:55.831+00:00 2026-10-15T03:39:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:56.836+00:00 2026-10-15T03:39:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:56.837+00:00 2026-10-15T03:39:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:57.841+00:00 2026-10-15T03:39:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:57.841+00:00 2026-10-15T03:39:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:58.846+00:00 2026-10-15T03:39:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:58.846+00:00 2026-10-15T03:39:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:39:59.850+00:00 2026-10-15T03:39:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:39:59.850+00:00 2026-10-15T03:39:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:00.855+00:00 2026-10-15T03:40:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:00.855+00:00 2026-10-15T03:40:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:01.861+00:00 2026-10-15T03:40:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:01.861+00:00 2026-10-15T03:40:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:02.867+00:00 2026-10-15T03:40:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:02.867+00:00 2026-10-15T03:40:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:03.871+00:00 2026-10-15T03:40:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:03.871+00:00 2026-10-15T03:40:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:04.876+00:00 2026-10-15T03:40:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:04.876+00:00 2026-10-15T03:40:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:05.933+00:00 2026-10-15T03:40:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:05.933+00:00 2026-10-15T03:40:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:06.886+00:00 2026-10-15T03:40:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:06.886+00:00 2026-10-15T03:40:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:07.890+00:00 2026-10-15T03:40:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:07.890+00:00 2026-10-15T03:40:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:08.895+00:00 2026-10-15T03:40:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:08.895+00:00 2026-10-15T03:40:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:09.900+00:00 2026-10-15T03:40:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:09.900+00:00 2026-10-15T03:40:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:10.939+00:00 2026-10-15T03:40:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:10.940+00:00 2026-10-15T03:40:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:11.910+00:00 2026-10-15T03:40:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:11.910+00:00 2026-10-15T03:40:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:12.915+00:00 2026-10-15T03:40:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:12.915+00:00 2026-10-15T03:40:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:13.920+00:00 2026-10-15T03:40:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:13.920+00:00 2026-10-15T03:40:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:14.927+00:00 2026-10-15T03:40:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:14.927+00:00 2026-10-15T03:40:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:15.930+00:00 2026-10-15T03:40:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:15.930+00:00 2026-10-15T03:40:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:16.942+00:00 2026-10-15T03:40:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:16.942+00:00 2026-10-15T03:40:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:17.942+00:00 2026-10-15T03:40:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:17.942+00:00 2026-10-15T03:40:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:18.945+00:00 2026-10-15T03:40:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:18.946+00:00 2026-10-15T03:40:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:19.953+00:00 2026-10-15T03:40:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:19.953+00:00 2026-10-15T03:40:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:20.956+00:00 2026-10-15T03:40:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:20.956+00:00 2026-10-15T03:40:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:22.010+00:00 2026-10-15T03:40:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:22.011+00:00 2026-10-15T03:40:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:22.966+00:00 2026-10-15T03:40:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:22.966+00:00 2026-10-15T03:40:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:23.972+00:00 2026-10-15T03:40:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:23.972+00:00 2026-10-15T03:40:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:24.976+00:00 2026-10-15T03:40:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:24.976+00:00 2026-10-15T03:40:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:25.981+00:00 2026-10-15T03:40:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:25.981+00:00 2026-10-15T03:40:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:26.985+00:00 2026-10-15T03:40:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:26.985+00:00 2026-10-15T03:40:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:27.990+00:00 2026-10-15T03:40:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:27.990+00:00 2026-10-15T03:40:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:28.995+00:00 2026-10-15T03:40:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:28.995+00:00 2026-10-15T03:40:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:30.003+00:00 2026-10-15T03:40:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:30.003+00:00 2026-10-15T03:40:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:31.005+00:00 2026-10-15T03:40:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:31.006+00:00 2026-10-15T03:40:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:32.012+00:00 2026-10-15T03:40:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:32.012+00:00 2026-10-15T03:40:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:33.017+00:00 2026-10-15T03:40:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:33.017+00:00 2026-10-15T03:40:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:34.022+00:00 2026-10-15T03:40:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:34.022+00:00 2026-10-15T03:40:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:35.027+00:00 2026-10-15T03:40:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:35.027+00:00 2026-10-15T03:40:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:36.033+00:00 2026-10-15T03:40:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:36.033+00:00 2026-10-15T03:40:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:37.038+00:00 2026-10-15T03:40:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:37.038+00:00 2026-10-15T03:40:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:38.043+00:00 2026-10-15T03:40:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:38.043+00:00 2026-10-15T03:40:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:39.048+00:00 2026-10-15T03:40:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:39.048+00:00 2026-10-15T03:40:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:40.053+00:00 2026-10-15T03:40:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:40.053+00:00 2026-10-15T03:40:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:41.057+00:00 2026-10-15T03:40:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:41.057+00:00 2026-10-15T03:40:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:42.063+00:00 2026-10-15T03:40:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:42.063+00:00 2026-10-15T03:40:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:43.093+00:00 2026-10-15T03:40:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:43.094+00:00 2026-10-15T03:40:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:44.080+00:00 2026-10-15T03:40:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:44.080+00:00 2026-10-15T03:40:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:45.078+00:00 2026-10-15T03:40:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:45.078+00:00 2026-10-15T03:40:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:46.083+00:00 2026-10-15T03:40:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:46.083+00:00 2026-10-15T03:40:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:47.089+00:00 2026-10-15T03:40:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:47.089+00:00 2026-10-15T03:40:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:48.098+00:00 2026-10-15T03:40:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:48.098+00:00 2026-10-15T03:40:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:49.097+00:00 2026-10-15T03:40:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:49.098+00:00 2026-10-15T03:40:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:50.103+00:00 2026-10-15T03:40:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:50.103+00:00 2026-10-15T03:40:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:51.221+00:00 2026-10-15T03:40:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:51.221+00:00 2026-10-15T03:40:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:52.121+00:00 2026-10-15T03:40:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:52.121+00:00 2026-10-15T03:40:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:53.121+00:00 2026-10-15T03:40:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:53.121+00:00 2026-10-15T03:40:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:54.126+00:00 2026-10-15T03:40:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:54.127+00:00 2026-10-15T03:40:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:55.130+00:00 2026-10-15T03:40:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:55.130+00:00 2026-10-15T03:40:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:56.230+00:00 2026-10-15T03:40:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:56.230+00:00 2026-10-15T03:40:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:57.141+00:00 2026-10-15T03:40:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:57.141+00:00 2026-10-15T03:40:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:58.146+00:00 2026-10-15T03:40:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:58.146+00:00 2026-10-15T03:40:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:40:59.152+00:00 2026-10-15T03:40:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:40:59.152+00:00 2026-10-15T03:40:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:00.162+00:00 2026-10-15T03:41:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:00.162+00:00 2026-10-15T03:41:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:01.162+00:00 2026-10-15T03:41:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:01.162+00:00 2026-10-15T03:41:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:02.166+00:00 2026-10-15T03:41:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:02.167+00:00 2026-10-15T03:41:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:03.172+00:00 2026-10-15T03:41:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:03.172+00:00 2026-10-15T03:41:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:04.176+00:00 2026-10-15T03:41:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:04.176+00:00 2026-10-15T03:41:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:05.181+00:00 2026-10-15T03:41:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:05.181+00:00 2026-10-15T03:41:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:06.187+00:00 2026-10-15T03:41:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:06.187+00:00 2026-10-15T03:41:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:07.192+00:00 2026-10-15T03:41:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:07.192+00:00 2026-10-15T03:41:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:08.197+00:00 2026-10-15T03:41:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:08.197+00:00 2026-10-15T03:41:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:09.200+00:00 2026-10-15T03:41:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:09.201+00:00 2026-10-15T03:41:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:10.206+00:00 2026-10-15T03:41:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:10.206+00:00 2026-10-15T03:41:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:11.211+00:00 2026-10-15T03:41:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:11.211+00:00 2026-10-15T03:41:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:12.216+00:00 2026-10-15T03:41:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:12.216+00:00 2026-10-15T03:41:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:13.221+00:00 2026-10-15T03:41:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:13.221+00:00 2026-10-15T03:41:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:14.227+00:00 2026-10-15T03:41:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:14.227+00:00 2026-10-15T03:41:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:15.232+00:00 2026-10-15T03:41:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:15.232+00:00 2026-10-15T03:41:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:16.236+00:00 2026-10-15T03:41:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:16.236+00:00 2026-10-15T03:41:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:17.241+00:00 2026-10-15T03:41:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:17.241+00:00 2026-10-15T03:41:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:18.247+00:00 2026-10-15T03:41:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:18.247+00:00 2026-10-15T03:41:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:19.251+00:00 2026-10-15T03:41:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:19.251+00:00 2026-10-15T03:41:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:20.257+00:00 2026-10-15T03:41:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:20.257+00:00 2026-10-15T03:41:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:21.261+00:00 2026-10-15T03:41:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:21.261+00:00 2026-10-15T03:41:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:22.267+00:00 2026-10-15T03:41:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:22.267+00:00 2026-10-15T03:41:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:23.272+00:00 2026-10-15T03:41:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:23.272+00:00 2026-10-15T03:41:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:24.276+00:00 2026-10-15T03:41:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:24.276+00:00 2026-10-15T03:41:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:25.283+00:00 2026-10-15T03:41:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:25.283+00:00 2026-10-15T03:41:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:26.286+00:00 2026-10-15T03:41:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:26.286+00:00 2026-10-15T03:41:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:27.292+00:00 2026-10-15T03:41:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:27.292+00:00 2026-10-15T03:41:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:28.297+00:00 2026-10-15T03:41:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:28.297+00:00 2026-10-15T03:41:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:29.303+00:00 2026-10-15T03:41:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:29.303+00:00 2026-10-15T03:41:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:30.308+00:00 2026-10-15T03:41:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:30.308+00:00 2026-10-15T03:41:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:31.314+00:00 2026-10-15T03:41:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:31.314+00:00 2026-10-15T03:41:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:32.319+00:00 2026-10-15T03:41:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:32.320+00:00 2026-10-15T03:41:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:33.324+00:00 2026-10-15T03:41:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:33.324+00:00 2026-10-15T03:41:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:34.329+00:00 2026-10-15T03:41:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:34.329+00:00 2026-10-15T03:41:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:35.334+00:00 2026-10-15T03:41:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:35.334+00:00 2026-10-15T03:41:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:36.346+00:00 2026-10-15T03:41:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:36.346+00:00 2026-10-15T03:41:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:37.344+00:00 2026-10-15T03:41:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:37.344+00:00 2026-10-15T03:41:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:38.349+00:00 2026-10-15T03:41:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:38.349+00:00 2026-10-15T03:41:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:39.353+00:00 2026-10-15T03:41:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:39.353+00:00 2026-10-15T03:41:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:40.358+00:00 2026-10-15T03:41:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:40.358+00:00 2026-10-15T03:41:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:41.363+00:00 2026-10-15T03:41:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:41.363+00:00 2026-10-15T03:41:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:42.369+00:00 2026-10-15T03:41:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:42.369+00:00 2026-10-15T03:41:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:43.373+00:00 2026-10-15T03:41:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:43.373+00:00 2026-10-15T03:41:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:44.378+00:00 2026-10-15T03:41:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:44.378+00:00 2026-10-15T03:41:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:45.383+00:00 2026-10-15T03:41:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:45.383+00:00 2026-10-15T03:41:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:46.388+00:00 2026-10-15T03:41:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:46.388+00:00 2026-10-15T03:41:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:47.392+00:00 2026-10-15T03:41:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:47.392+00:00 2026-10-15T03:41:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:48.397+00:00 2026-10-15T03:41:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:48.397+00:00 2026-10-15T03:41:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:49.404+00:00 2026-10-15T03:41:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:49.404+00:00 2026-10-15T03:41:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:50.407+00:00 2026-10-15T03:41:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:50.407+00:00 2026-10-15T03:41:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:51.411+00:00 2026-10-15T03:41:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:51.412+00:00 2026-10-15T03:41:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:52.417+00:00 2026-10-15T03:41:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:52.417+00:00 2026-10-15T03:41:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:53.422+00:00 2026-10-15T03:41:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:53.422+00:00 2026-10-15T03:41:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:54.427+00:00 2026-10-15T03:41:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:54.427+00:00 2026-10-15T03:41:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:55.433+00:00 2026-10-15T03:41:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:55.433+00:00 2026-10-15T03:41:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:56.455+00:00 2026-10-15T03:41:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:56.455+00:00 2026-10-15T03:41:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:57.443+00:00 2026-10-15T03:41:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:57.443+00:00 2026-10-15T03:41:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:58.447+00:00 2026-10-15T03:41:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:58.447+00:00 2026-10-15T03:41:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:41:59.453+00:00 2026-10-15T03:41:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:41:59.454+00:00 2026-10-15T03:41:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:00.457+00:00 2026-10-15T03:42:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:00.457+00:00 2026-10-15T03:42:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:01.486+00:00 2026-10-15T03:42:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:01.486+00:00 2026-10-15T03:42:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:02.468+00:00 2026-10-15T03:42:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:02.468+00:00 2026-10-15T03:42:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:03.473+00:00 2026-10-15T03:42:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:03.473+00:00 2026-10-15T03:42:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:04.477+00:00 2026-10-15T03:42:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:04.477+00:00 2026-10-15T03:42:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:05.483+00:00 2026-10-15T03:42:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:05.483+00:00 2026-10-15T03:42:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:06.487+00:00 2026-10-15T03:42:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:06.487+00:00 2026-10-15T03:42:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:07.492+00:00 2026-10-15T03:42:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:07.492+00:00 2026-10-15T03:42:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:08.498+00:00 2026-10-15T03:42:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:08.499+00:00 2026-10-15T03:42:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:09.502+00:00 2026-10-15T03:42:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:09.502+00:00 2026-10-15T03:42:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:10.512+00:00 2026-10-15T03:42:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:10.512+00:00 2026-10-15T03:42:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:11.513+00:00 2026-10-15T03:42:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:11.513+00:00 2026-10-15T03:42:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:12.518+00:00 2026-10-15T03:42:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:12.518+00:00 2026-10-15T03:42:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:13.523+00:00 2026-10-15T03:42:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:13.523+00:00 2026-10-15T03:42:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:14.528+00:00 2026-10-15T03:42:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:14.528+00:00 2026-10-15T03:42:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:15.535+00:00 2026-10-15T03:42:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:15.535+00:00 2026-10-15T03:42:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:16.538+00:00 2026-10-15T03:42:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:16.538+00:00 2026-10-15T03:42:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:17.547+00:00 2026-10-15T03:42:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:17.547+00:00 2026-10-15T03:42:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:18.550+00:00 2026-10-15T03:42:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:18.550+00:00 2026-10-15T03:42:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:19.566+00:00 2026-10-15T03:42:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:19.566+00:00 2026-10-15T03:42:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:20.574+00:00 2026-10-15T03:42:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:20.574+00:00 2026-10-15T03:42:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:21.568+00:00 2026-10-15T03:42:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:21.568+00:00 2026-10-15T03:42:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:22.573+00:00 2026-10-15T03:42:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:22.574+00:00 2026-10-15T03:42:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:23.586+00:00 2026-10-15T03:42:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:23.586+00:00 2026-10-15T03:42:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:24.583+00:00 2026-10-15T03:42:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:24.584+00:00 2026-10-15T03:42:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:25.589+00:00 2026-10-15T03:42:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:25.589+00:00 2026-10-15T03:42:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:26.594+00:00 2026-10-15T03:42:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:26.594+00:00 2026-10-15T03:42:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:27.599+00:00 2026-10-15T03:42:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:27.599+00:00 2026-10-15T03:42:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:28.610+00:00 2026-10-15T03:42:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:28.610+00:00 2026-10-15T03:42:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:29.610+00:00 2026-10-15T03:42:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:29.610+00:00 2026-10-15T03:42:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:30.615+00:00 2026-10-15T03:42:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:30.615+00:00 2026-10-15T03:42:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:31.620+00:00 2026-10-15T03:42:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:31.621+00:00 2026-10-15T03:42:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:32.626+00:00 2026-10-15T03:42:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:32.626+00:00 2026-10-15T03:42:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:33.646+00:00 2026-10-15T03:42:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:33.646+00:00 2026-10-15T03:42:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:34.636+00:00 2026-10-15T03:42:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:34.636+00:00 2026-10-15T03:42:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:35.641+00:00 2026-10-15T03:42:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:35.641+00:00 2026-10-15T03:42:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:36.647+00:00 2026-10-15T03:42:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:36.647+00:00 2026-10-15T03:42:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:37.654+00:00 2026-10-15T03:42:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:37.654+00:00 2026-10-15T03:42:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:38.657+00:00 2026-10-15T03:42:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:38.657+00:00 2026-10-15T03:42:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:39.672+00:00 2026-10-15T03:42:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:39.672+00:00 2026-10-15T03:42:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:40.667+00:00 2026-10-15T03:42:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:40.667+00:00 2026-10-15T03:42:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:41.673+00:00 2026-10-15T03:42:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:41.673+00:00 2026-10-15T03:42:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:42.676+00:00 2026-10-15T03:42:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:42.676+00:00 2026-10-15T03:42:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:43.681+00:00 2026-10-15T03:42:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:43.682+00:00 2026-10-15T03:42:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:44.686+00:00 2026-10-15T03:42:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:44.687+00:00 2026-10-15T03:42:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:45.691+00:00 2026-10-15T03:42:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:45.691+00:00 2026-10-15T03:42:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:46.698+00:00 2026-10-15T03:42:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:46.698+00:00 2026-10-15T03:42:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:47.702+00:00 2026-10-15T03:42:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:47.702+00:00 2026-10-15T03:42:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:48.708+00:00 2026-10-15T03:42:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:48.708+00:00 2026-10-15T03:42:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:49.712+00:00 2026-10-15T03:42:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:49.712+00:00 2026-10-15T03:42:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:50.721+00:00 2026-10-15T03:42:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:50.721+00:00 2026-10-15T03:42:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:51.728+00:00 2026-10-15T03:42:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:51.728+00:00 2026-10-15T03:42:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:52.727+00:00 2026-10-15T03:42:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:52.727+00:00 2026-10-15T03:42:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:53.732+00:00 2026-10-15T03:42:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:53.732+00:00 2026-10-15T03:42:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:54.737+00:00 2026-10-15T03:42:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:54.737+00:00 2026-10-15T03:42:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:55.742+00:00 2026-10-15T03:42:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:55.742+00:00 2026-10-15T03:42:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:56.747+00:00 2026-10-15T03:42:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:56.747+00:00 2026-10-15T03:42:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:57.762+00:00 2026-10-15T03:42:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:57.762+00:00 2026-10-15T03:42:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:58.757+00:00 2026-10-15T03:42:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:58.757+00:00 2026-10-15T03:42:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:42:59.762+00:00 2026-10-15T03:42:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:42:59.762+00:00 2026-10-15T03:42:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:00.773+00:00 2026-10-15T03:43:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:00.773+00:00 2026-10-15T03:43:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:01.776+00:00 2026-10-15T03:43:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:01.776+00:00 2026-10-15T03:43:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:02.777+00:00 2026-10-15T03:43:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:02.777+00:00 2026-10-15T03:43:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:03.782+00:00 2026-10-15T03:43:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:03.782+00:00 2026-10-15T03:43:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:04.789+00:00 2026-10-15T03:43:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:04.789+00:00 2026-10-15T03:43:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:05.792+00:00 2026-10-15T03:43:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:05.792+00:00 2026-10-15T03:43:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:06.796+00:00 2026-10-15T03:43:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:06.796+00:00 2026-10-15T03:43:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:07.802+00:00 2026-10-15T03:43:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:07.802+00:00 2026-10-15T03:43:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:08.806+00:00 2026-10-15T03:43:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:08.806+00:00 2026-10-15T03:43:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:09.810+00:00 2026-10-15T03:43:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:09.810+00:00 2026-10-15T03:43:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:10.816+00:00 2026-10-15T03:43:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:10.816+00:00 2026-10-15T03:43:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:11.822+00:00 2026-10-15T03:43:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:11.822+00:00 2026-10-15T03:43:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:12.826+00:00 2026-10-15T03:43:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:12.826+00:00 2026-10-15T03:43:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:13.831+00:00 2026-10-15T03:43:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:13.831+00:00 2026-10-15T03:43:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:14.836+00:00 2026-10-15T03:43:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:14.836+00:00 2026-10-15T03:43:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:15.842+00:00 2026-10-15T03:43:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:15.842+00:00 2026-10-15T03:43:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:43:16.847+00:00 2026-10-15T03:43:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:43:16.847+00:00 2026-10-15T03:43:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.694+00:00 2026-10-15T03:43:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.695+00:00 2026-10-15T03:43:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.695+00:00 2026-10-15T03:43:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.695+00:00 2026-10-15T03:43:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.749+00:00 2026-10-15T03:43:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.749+00:00 2026-10-15T03:43:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.806+00:00 2026-10-15T03:43:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.806+00:00 2026-10-15T03:43:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.813+00:00 2026-10-15T03:43:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.813+00:00 2026-10-15T03:43:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.851+00:00 2026-10-15T03:43:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.851+00:00 2026-10-15T03:43:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.861+00:00 2026-10-15T03:43:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.861+00:00 2026-10-15T03:43:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.902+00:00 2026-10-15T03:43:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.902+00:00 2026-10-15T03:43:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.951+00:00 2026-10-15T03:43:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.951+00:00 2026-10-15T03:43:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.002+00:00 2026-10-15T03:43:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.002+00:00 2026-10-15T03:43:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.062+00:00 2026-10-15T03:43:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.062+00:00 2026-10-15T03:43:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.185+00:00 2026-10-15T03:43:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.292+00:00 2026-10-15T03:45:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.292+00:00 2026-10-15T03:45:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:46:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:46:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:46:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.393+00:00 2026-10-15T03:48:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.393+00:00 2026-10-15T03:48:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.393+00:00 2026-10-15T03:48:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:05.614+00:00 2026-10-15T03:49:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:06.622+00:00 2026-10-15T03:49:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:07.626+00:00 2026-10-15T03:49:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:08.631+00:00 2026-10-15T03:49:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:09.637+00:00 2026-10-15T03:49:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:10.643+00:00 2026-10-15T03:49:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:11.646+00:00 2026-10-15T03:49:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:12.651+00:00 2026-10-15T03:49:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:13.657+00:00 2026-10-15T03:49:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:14.661+00:00 2026-10-15T03:49:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:15.666+00:00 2026-10-15T03:49:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:16.671+00:00 2026-10-15T03:49:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T03:49:17.676+00:00 2026-10-15T03:49:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:05:57.232+00:00 E1014 21:05:57.231723 5403 v2.go:150] "Unhandled Error" err="next reader: read tcp 192.168.1.37:57575->136.116.158.163:443: read: connection reset by peer" +2026-10-15T04:05:57.233+00:00 E1014 21:05:57.231748 5403 v2.go:167] "Unhandled Error" err="next reader: read tcp 192.168.1.37:57575->136.116.158.163:443: read: connection reset by peer" +2026-10-15T04:05:57.234+00:00 error: error reading from error stream: next reader: read tcp 192.168.1.37:57575->136.116.158.163:443: read: connection reset by peer diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log new file mode 100644 index 00000000..30f82985 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log @@ -0,0 +1,36 @@ +watching; recorders [5401, 5402, 5404, 5406, 5408, 5403, 5405, 5407] write into runs/2026-10-15_cold_start/D-8b-h100-cached +collected 0 worker log(s) into runs/2026-10-15_cold_start/D-8b-h100-cached +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | unmeasured | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | unmeasured | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | unmeasured | +| container create | trainer.event_Pulled | trainer.container_started | unmeasured | +| process start to load start (imports) | trainer.container_started | trainer.load_start | unmeasured | +| weights load | trainer.load_start | trainer.load_done | unmeasured | +| ready key | trainer.load_done | trainer.ready_key | unmeasured | +| client: create_model blocked | client.create_model.start | client.create_model.done | 555.2 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | unmeasured | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | unmeasured | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | unmeasured | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | unmeasured | +| container create | sampler.event_Pulled | sampler.container_started | unmeasured | +| process start to first log | sampler.container_started | sampler.process_first_log | unmeasured | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | unmeasured | +| engine init until load start | sampler.engine_init_start | sampler.load_start | unmeasured | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | unmeasured | +| torch.compile | sampler.weights_loaded | sampler.compile_done | unmeasured | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | unmeasured | +| ready key | sampler.engine_init_done | sampler.ready_key | unmeasured | +| client: first sample blocked | client.sample.start | client.sample.done | unmeasured | + +DONE=0 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log new file mode 100644 index 00000000..8a48c226 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T03:39:04.606+00:00 2026-10-15T03:39:04.595116714Z {"level":"info","ts":"2026-10-15T03:39:04Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"991ac44f-13dc-4b3b-88a3-ea3017a1451b","claim":"claim-lora-qwen-qwen3-8b-0-trainer-f4ace73f","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} +2026-10-15T03:48:27.398+00:00 error: read tcp 192.168.1.37:57571->136.116.158.163:443: read: connection reset by peer diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json new file mode 100644 index 00000000..66e85a7f --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json @@ -0,0 +1,26 @@ +{ + "case": "D-8b-h100-cached", + "collected": "2026-10-15T04:27:13.680+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 03:39:03.686000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen3-8B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 03:39:03.686000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 03:48:18.935000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "ddb4615c-4bb0-44b0-92a1-c79e668c7324" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 03:48:21.731000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": {} +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.md b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.md new file mode 100644 index 00000000..46774e8d --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.md @@ -0,0 +1,54 @@ +# Cold start timeline: D-8b-h100-cached + +t=0 is the client's `create_model` call at 2026-10-15T03:39:03.686+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer + +no worker observed + +## sampler + +no worker observed + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 03:39:03.686 | 0.0 | +| client.create_model.start | 03:39:03.686 | 0.0 | +| client.create_model.done | 03:48:18.935 | 555.2 | +| client.forward_backward.start | 03:48:21.731 | 558.0 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | unmeasured | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | unmeasured | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | unmeasured | +| container create | trainer.event_Pulled | trainer.container_started | unmeasured | +| process start to load start (imports) | trainer.container_started | trainer.load_start | unmeasured | +| weights load | trainer.load_start | trainer.load_done | unmeasured | +| ready key | trainer.load_done | trainer.ready_key | unmeasured | +| client: create_model blocked | client.create_model.start | client.create_model.done | 555.2 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | unmeasured | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | unmeasured | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | unmeasured | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | unmeasured | +| container create | sampler.event_Pulled | sampler.container_started | unmeasured | +| process start to first log | sampler.container_started | sampler.process_first_log | unmeasured | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | unmeasured | +| engine init until load start | sampler.engine_init_start | sampler.load_start | unmeasured | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | unmeasured | +| torch.compile | sampler.weights_loaded | sampler.compile_done | unmeasured | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | unmeasured | +| ready key | sampler.engine_init_done | sampler.ready_key | unmeasured | +| client: first sample blocked | client.sample.start | client.sample.done | unmeasured | + diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt new file mode 100644 index 00000000..722e1311 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T03:38:59.461+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt new file mode 100644 index 00000000..0d162c07 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json new file mode 100644 index 00000000..6da8d279 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json @@ -0,0 +1,8 @@ +{ + "apiVersion": "v1", + "items": [], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached.run.log b/runs/2026-10-15_cold_start/D-8b-h100-cached.run.log new file mode 100644 index 00000000..1024f918 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached.run.log @@ -0,0 +1,35 @@ +watching; recorders [8878, 8879, 8881, 8883, 8885, 8880, 8882, 8884] write into runs/2026-10-15_cold_start/D-8b-h100-cached +collected 2 worker log(s) into runs/2026-10-15_cold_start/D-8b-h100-cached +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.2 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.4 | +| weights load | trainer.load_start | trainer.load_done | 39.9 | +| ready key | trainer.load_done | trainer.ready_key | -0.3 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 52.2 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.3 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 3.0 | +| container create | sampler.event_Pulled | sampler.container_started | 1.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 7.1 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 17.7 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 13.5 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.4 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.5 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.6 | +| client: first sample blocked | client.sample.start | client.sample.done | 120.7 | + diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log new file mode 100644 index 00000000..b2279560 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log @@ -0,0 +1,36 @@ +2026-10-15T04:28:48.050+00:00 2026-10-15T04:28:48.048635307Z INFO: 127.0.0.1:39700 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T04:28:48.286+00:00 2026-10-15T04:28:48.286095440Z INFO: 127.0.0.1:39712 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T04:28:48.355+00:00 2026-10-15T04:28:48.350028924Z INFO: 127.0.0.1:39712 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T04:28:48.592+00:00 2026-10-15T04:28:48.593057216Z [API_SERVER] enqueued op=create_model request_id=db28c229-9bd6-4845-bd41-edb2e3018375 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 +2026-10-15T04:28:48.594+00:00 2026-10-15T04:28:48.593584627Z INFO: 127.0.0.1:39714 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T04:28:53.965+00:00 2026-10-15T04:28:53.964292215Z INFO: 10.32.3.1:47308 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:28:58.571+00:00 2026-10-15T04:28:58.568532635Z INFO: 127.0.0.1:59700 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:29:03.967+00:00 2026-10-15T04:29:03.965572350Z INFO: 10.32.3.1:49272 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:29:13.967+00:00 2026-10-15T04:29:13.965469203Z INFO: 10.32.3.1:53736 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:29:23.972+00:00 2026-10-15T04:29:23.965165004Z INFO: 10.32.3.1:57264 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:29:33.967+00:00 2026-10-15T04:29:33.965503320Z INFO: 10.32.3.1:41922 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:29:38.739+00:00 2026-10-15T04:29:38.738456425Z INFO: 127.0.0.1:58346 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:29:40.123+00:00 2026-10-15T04:29:40.120084777Z INFO: 127.0.0.1:58376 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T04:29:42.785+00:00 2026-10-15T04:29:42.783707482Z [API_SERVER] enqueued op=forward_backward request_id=5827dd5b-d046-465a-9f51-a4c5fb10be88 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 +2026-10-15T04:29:42.786+00:00 2026-10-15T04:29:42.784716161Z INFO: 127.0.0.1:58376 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T04:29:43.964+00:00 2026-10-15T04:29:43.964611788Z INFO: 10.32.3.1:36414 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:29:45.701+00:00 2026-10-15T04:29:45.690175181Z [API_SERVER] enqueued op=optim_step request_id=0bd1fc1f-9862-42b8-87a3-bb2b38d15a08 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 +2026-10-15T04:29:45.707+00:00 2026-10-15T04:29:45.690965153Z INFO: 127.0.0.1:58376 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T04:29:47.841+00:00 2026-10-15T04:29:47.840662196Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=bb1b8120-ce82-4ed2-8a6c-e0803b9cdc65 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 +2026-10-15T04:29:47.841+00:00 2026-10-15T04:29:47.841190565Z INFO: 127.0.0.1:58376 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T04:29:49.846+00:00 2026-10-15T04:29:49.845879451Z INFO: 127.0.0.1:35416 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T04:29:53.966+00:00 2026-10-15T04:29:53.965198781Z INFO: 10.32.3.1:35352 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:30:03.968+00:00 2026-10-15T04:30:03.966186951Z INFO: 10.32.3.1:35492 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:30:13.967+00:00 2026-10-15T04:30:13.965413981Z INFO: 10.32.3.1:42874 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:30:23.995+00:00 2026-10-15T04:30:23.965280850Z INFO: 10.32.3.1:52978 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:30:33.965+00:00 2026-10-15T04:30:33.964533329Z INFO: 10.32.3.1:52066 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:30:38.938+00:00 2026-10-15T04:30:38.922489978Z INFO: 127.0.0.1:36472 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:30:43.967+00:00 2026-10-15T04:30:43.965005983Z INFO: 10.32.3.1:41578 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:30:53.966+00:00 2026-10-15T04:30:53.964647584Z INFO: 10.32.3.1:37766 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:31:03.966+00:00 2026-10-15T04:31:03.964948698Z INFO: 10.32.3.1:52962 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:31:13.967+00:00 2026-10-15T04:31:13.964286630Z INFO: 10.32.3.1:39666 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:31:23.967+00:00 2026-10-15T04:31:23.965294376Z INFO: 10.32.3.1:35434 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:31:29.132+00:00 2026-10-15T04:31:29.129760597Z INFO: 127.0.0.1:44156 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:31:33.966+00:00 2026-10-15T04:31:33.965082373Z INFO: 10.32.3.1:36136 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:31:43.967+00:00 2026-10-15T04:31:43.964688954Z INFO: 10.32.3.1:47940 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:31:53.966+00:00 2026-10-15T04:31:53.964597059Z INFO: 10.32.3.1:57990 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl b/runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl new file mode 100644 index 00000000..7a8c61f7 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T04:28:47.756+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T04:28:47.756+00:00", "t": 0.0, "phase": "create_model.start"} +{"ts": "2026-10-15T04:29:39.925+00:00", "t": 52.169, "phase": "create_model.done", "model_id": "db28c229-9bd6-4845-bd41-edb2e3018375"} +{"ts": "2026-10-15T04:29:42.710+00:00", "t": 54.955, "phase": "forward_backward.start"} +{"ts": "2026-10-15T04:29:45.623+00:00", "t": 57.867, "phase": "forward_backward.done"} +{"ts": "2026-10-15T04:29:47.732+00:00", "t": 59.976, "phase": "optim_step.done"} +{"ts": "2026-10-15T04:29:47.733+00:00", "t": 59.977, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T04:29:49.654+00:00", "t": 61.898, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T04:29:49.657+00:00", "t": 61.901, "phase": "sample.start"} +{"ts": "2026-10-15T04:31:50.315+00:00", "t": 182.559, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} +{"ts": "2026-10-15T04:31:50.316+00:00", "t": 182.56, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/client.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/client.log new file mode 100644 index 00000000..7c7e9887 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.000s create_model.start +[cold-start] 52.169s create_model.done {'model_id': 'db28c229-9bd6-4845-bd41-edb2e3018375'} +[cold-start] 54.955s forward_backward.start +[cold-start] 57.867s forward_backward.done +[cold-start] 59.976s optim_step.done +[cold-start] 59.977s save_weights_and_get_sampling_client.start +[cold-start] 61.898s save_weights_and_get_sampling_client.done +[cold-start] 61.901s sample.start +[cold-start] 182.559s sample.done {'text': ' 4\n\nQuestion: What is '} +[cold-start] 182.560s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt new file mode 100644 index 00000000..446608ec --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T04:31:50.787+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/events.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/events.json new file mode 100644 index 00000000..6386b858 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/events.json @@ -0,0 +1,661 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:31:11Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071080495023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:31:11Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T03:31:11Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96671460e727", + "namespace": "openrl-system", + "resourceVersion": "1792035071245135014", + "uid": "8acbbf2e-49f9-4202-949a-77c580789a9a" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:31:15Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:31:15Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", + "metadata": { + "creationTimestamp": "2026-10-15T03:31:15Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9668063dcb02", + "namespace": "openrl-system", + "resourceVersion": "1792035075278831019", + "uid": "6912eb66-4b94-4b18-a74e-ab327aeddb23" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:32:50Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:32:50Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m35.306s (1m35.306s including waiting). Image size: 7123551739 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T03:32:50Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e36f0cbc2", + "namespace": "openrl-system", + "resourceVersion": "1792035170612415015", + "uid": "8591c4e1-a15e-4174-ad4a-fd97a845ccf1" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:32:50Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:32:50Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T03:32:50Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3778892c", + "namespace": "openrl-system", + "resourceVersion": "1792035170630543005", + "uid": "ffa2ae1f-d67a-48f5-bbae-e60a320c7ca4" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:32:50Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:32:50Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T03:32:50Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3f40302b", + "namespace": "openrl-system", + "resourceVersion": "1792035170729087005", + "uid": "1519861d-65be-4db6-b777-bd8220ced5f8" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:38:15Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792035071201999023", + "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:38:15Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:38:15Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96c9e6a9a8f5", + "namespace": "openrl-system", + "resourceVersion": "1792035495672207024", + "uid": "ddca643d-0c58-4430-9459-5686015bcf61" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:38:15Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792034054934527006", + "uid": "5f7199d4-14aa-4747-9728-d940f5af679f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:38:15Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:38:15Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96c9ee8c225c", + "namespace": "openrl-system", + "resourceVersion": "1792035495812399014", + "uid": "ea4fdc32-2c46-41b7-8537-f60b2a3e7ce3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:04Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544745343006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:04Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:04Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d55baf7ea2", + "namespace": "openrl-system", + "resourceVersion": "1792035544868287013", + "uid": "717619c9-bc6f-4410-89cb-933289a9091a" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:08Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:08Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:08Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d6291f5cc5", + "namespace": "openrl-system", + "resourceVersion": "1792035548309503024", + "uid": "0ad1918b-487b-45d0-95f6-90606fb0f996" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:08Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:08Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:08Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d629aa8317", + "namespace": "openrl-system", + "resourceVersion": "1792035548331599007", + "uid": "a146313b-77f3-450f-9b9d-6d0b1f04c6cf" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:39:08Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:39:08Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T03:39:08Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d62f39a7f5", + "namespace": "openrl-system", + "resourceVersion": "1792035548412447000", + "uid": "fcf01033-06e2-4cbd-a2c7-51bf347ddf42" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T03:45:24Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792035544844159006", + "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T03:45:24Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T03:45:24Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de972dcf359e7a", + "namespace": "openrl-system", + "resourceVersion": "1792035924784175002", + "uid": "b8207d84-11ad-4a25-867c-7be9b78ba580" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt new file mode 100644 index 00000000..3a77965e --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt @@ -0,0 +1,223 @@ +2026-10-15T04:29:59.120383793Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T04:30:07.125364996Z INFO 10-15 04:30:07 [model.py:692] Resolved architecture: Qwen3ForCausalLM +2026-10-15T04:30:07.125609457Z INFO 10-15 04:30:07 [model.py:2030] Using max model len 8192 +2026-10-15T04:30:07.127331114Z INFO 10-15 04:30:07 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. +2026-10-15T04:30:07.128167407Z INFO 10-15 04:30:07 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T04:30:09.663951222Z WARNING 10-15 04:30:09 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T04:30:16.823040941Z (EngineCore pid=169) INFO 10-15 04:30:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T04:30:17.164644024Z (EngineCore pid=169) INFO 10-15 04:30:17 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_340dc35f3b0242e2bdbc2b172fcc048b backend=nccl +2026-10-15T04:30:17.168650276Z (EngineCore pid=169) INFO 10-15 04:30:17 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T04:30:17.168683746Z (EngineCore pid=169) INFO 10-15 04:30:17 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T04:30:17.937691413Z (EngineCore pid=169) INFO 10-15 04:30:17 [model_runner.py:396] Loading model from scratch... +2026-10-15T04:30:18.521439941Z (EngineCore pid=169) INFO 10-15 04:30:18 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T04:30:18.521753371Z (EngineCore pid=169) INFO 10-15 04:30:18 [flash_attn.py:1116] Using FlashAttention version 3 +2026-10-15T04:30:18.637928137Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T04:30:18.638073197Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T04:30:18.941812484Z (EngineCore pid=169) INFO 10-15 04:30:18 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. +2026-10-15T04:30:18.941839205Z (EngineCore pid=169) INFO 10-15 04:30:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T04:30:18.942611568Z (EngineCore pid=169) +Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T04:28:57.227937676Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T04:28:57.227957423Z +2026-10-15T04:28:57.431621575Z Loading base model Qwen/Qwen3-8B to cuda:0... +2026-10-15T04:28:57.548837029Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T04:29:37.083323739Z +Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json new file mode 100644 index 00000000..86b1422e --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-api-server-645ff9777d-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "645ff9777d", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + } + ], + "resourceVersion": "1791941173996783021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-79xdl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:02Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:26:13Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:26:02Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-79xdl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.10", + "podIPs": [ + { + "ip": "10.32.3.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-sampler", + "uid": "aac5fdd5-1826-49c7-8392-5165024b0a69" + } + ], + "resourceVersion": "1792038592210623023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-sampler-aac5fdd5" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "64aee2d3-084c-46a4-9c77-df7f6442e580" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-vvjnl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-vvjnl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:29:52Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:29:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:29:52Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:29:52Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:29:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "40Gi" + }, + "containerID": "containerd://cb24f70562bc53e327a3c69f58bcfb325486c128f44f76ed9a396e258881cab1", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:29:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-vvjnl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.1.8", + "podIPs": [ + { + "ip": "10.32.1.8" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:29:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-trainer", + "uid": "cde28d2b-7497-4559-ae79-2c350558ddd1" + } + ], + "resourceVersion": "1792038533321423006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-trainer-cde28d2b" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "6521caef-db14-4d7b-8477-e644e69919ee" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2wss8", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2wss8", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:28:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:28:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "36Gi" + }, + "containerID": "containerd://c9a7c710fac76fa5cc69dea265bf295ddd44ae0adc3cab5d45f3adca2265857b", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2wss8", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.4.8", + "podIPs": [ + { + "ip": "10.32.4.8" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:28:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log new file mode 100644 index 00000000..4235b80d --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log @@ -0,0 +1,147 @@ +2026-10-15T04:29:37.911+00:00 2026-10-15T04:29:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:38.916+00:00 2026-10-15T04:29:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:39.919+00:00 2026-10-15T04:29:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:40.924+00:00 2026-10-15T04:29:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:41.929+00:00 2026-10-15T04:29:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:42.941+00:00 2026-10-15T04:29:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:43.939+00:00 2026-10-15T04:29:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:44.944+00:00 2026-10-15T04:29:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:45.950+00:00 2026-10-15T04:29:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:46.953+00:00 2026-10-15T04:29:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:47.959+00:00 2026-10-15T04:29:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:48.964+00:00 2026-10-15T04:29:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:49.969+00:00 2026-10-15T04:29:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:50.975+00:00 2026-10-15T04:29:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:51.981+00:00 2026-10-15T04:29:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:52.993+00:00 2026-10-15T04:29:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:53.990+00:00 2026-10-15T04:29:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:54.995+00:00 2026-10-15T04:29:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:56.000+00:00 2026-10-15T04:29:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:57.008+00:00 2026-10-15T04:29:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:58.011+00:00 2026-10-15T04:29:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:29:59.081+00:00 2026-10-15T04:29:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:00.021+00:00 2026-10-15T04:30:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:01.026+00:00 2026-10-15T04:30:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:02.033+00:00 2026-10-15T04:30:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:03.037+00:00 2026-10-15T04:30:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:04.044+00:00 2026-10-15T04:30:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:05.047+00:00 2026-10-15T04:30:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:06.053+00:00 2026-10-15T04:30:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:07.058+00:00 2026-10-15T04:30:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:08.064+00:00 2026-10-15T04:30:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:09.069+00:00 2026-10-15T04:30:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:10.074+00:00 2026-10-15T04:30:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:11.079+00:00 2026-10-15T04:30:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:12.099+00:00 2026-10-15T04:30:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:13.090+00:00 2026-10-15T04:30:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:14.095+00:00 2026-10-15T04:30:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:15.108+00:00 2026-10-15T04:30:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:16.106+00:00 2026-10-15T04:30:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:17.114+00:00 2026-10-15T04:30:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:18.116+00:00 2026-10-15T04:30:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:19.122+00:00 2026-10-15T04:30:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:20.127+00:00 2026-10-15T04:30:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:21.132+00:00 2026-10-15T04:30:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:22.138+00:00 2026-10-15T04:30:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:23.143+00:00 2026-10-15T04:30:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:24.148+00:00 2026-10-15T04:30:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:25.153+00:00 2026-10-15T04:30:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:26.157+00:00 2026-10-15T04:30:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:27.162+00:00 2026-10-15T04:30:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:28.167+00:00 2026-10-15T04:30:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:29.171+00:00 2026-10-15T04:30:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:30.177+00:00 2026-10-15T04:30:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:31.182+00:00 2026-10-15T04:30:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:32.187+00:00 2026-10-15T04:30:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:33.195+00:00 2026-10-15T04:30:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:34.196+00:00 2026-10-15T04:30:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:35.202+00:00 2026-10-15T04:30:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:36.207+00:00 2026-10-15T04:30:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:37.212+00:00 2026-10-15T04:30:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:38.218+00:00 2026-10-15T04:30:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:39.223+00:00 2026-10-15T04:30:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:40.228+00:00 2026-10-15T04:30:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:41.234+00:00 2026-10-15T04:30:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:42.238+00:00 2026-10-15T04:30:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:43.245+00:00 2026-10-15T04:30:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:44.258+00:00 2026-10-15T04:30:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:45.254+00:00 2026-10-15T04:30:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:46.303+00:00 2026-10-15T04:30:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:47.268+00:00 2026-10-15T04:30:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:48.271+00:00 2026-10-15T04:30:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:49.275+00:00 2026-10-15T04:30:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:50.280+00:00 2026-10-15T04:30:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:51.286+00:00 2026-10-15T04:30:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:52.291+00:00 2026-10-15T04:30:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:53.296+00:00 2026-10-15T04:30:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:54.301+00:00 2026-10-15T04:30:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:55.304+00:00 2026-10-15T04:30:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:56.309+00:00 2026-10-15T04:30:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:57.314+00:00 2026-10-15T04:30:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:58.319+00:00 2026-10-15T04:30:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:30:59.415+00:00 2026-10-15T04:30:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:00.329+00:00 2026-10-15T04:31:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:01.334+00:00 2026-10-15T04:31:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:02.339+00:00 2026-10-15T04:31:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:03.344+00:00 2026-10-15T04:31:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:04.349+00:00 2026-10-15T04:31:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:05.354+00:00 2026-10-15T04:31:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:06.358+00:00 2026-10-15T04:31:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:07.363+00:00 2026-10-15T04:31:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:08.368+00:00 2026-10-15T04:31:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:09.382+00:00 2026-10-15T04:31:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:10.377+00:00 2026-10-15T04:31:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:11.382+00:00 2026-10-15T04:31:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:12.388+00:00 2026-10-15T04:31:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:13.392+00:00 2026-10-15T04:31:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:14.397+00:00 2026-10-15T04:31:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:15.401+00:00 2026-10-15T04:31:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:16.407+00:00 2026-10-15T04:31:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:17.412+00:00 2026-10-15T04:31:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:18.416+00:00 2026-10-15T04:31:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:19.423+00:00 2026-10-15T04:31:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:20.427+00:00 2026-10-15T04:31:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:21.431+00:00 2026-10-15T04:31:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:22.437+00:00 2026-10-15T04:31:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:23.454+00:00 2026-10-15T04:31:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:24.447+00:00 2026-10-15T04:31:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:25.452+00:00 2026-10-15T04:31:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:26.456+00:00 2026-10-15T04:31:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:27.461+00:00 2026-10-15T04:31:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:28.466+00:00 2026-10-15T04:31:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:29.472+00:00 2026-10-15T04:31:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:30.479+00:00 2026-10-15T04:31:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:31.482+00:00 2026-10-15T04:31:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:32.487+00:00 2026-10-15T04:31:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:33.494+00:00 2026-10-15T04:31:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:34.498+00:00 2026-10-15T04:31:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:35.503+00:00 2026-10-15T04:31:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:36.512+00:00 2026-10-15T04:31:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:37.517+00:00 2026-10-15T04:31:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:38.518+00:00 2026-10-15T04:31:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:39.523+00:00 2026-10-15T04:31:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:40.528+00:00 2026-10-15T04:31:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:41.532+00:00 2026-10-15T04:31:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:42.537+00:00 2026-10-15T04:31:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:43.543+00:00 2026-10-15T04:31:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:44.547+00:00 2026-10-15T04:31:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:45.561+00:00 2026-10-15T04:31:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:46.559+00:00 2026-10-15T04:31:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:47.570+00:00 2026-10-15T04:31:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:48.578+00:00 2026-10-15T04:31:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:49.574+00:00 2026-10-15T04:31:49Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:49.574+00:00 2026-10-15T04:31:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:50.579+00:00 2026-10-15T04:31:50Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:50.579+00:00 2026-10-15T04:31:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:51.585+00:00 2026-10-15T04:31:51Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:51.585+00:00 2026-10-15T04:31:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:52.592+00:00 2026-10-15T04:31:52Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:52.592+00:00 2026-10-15T04:31:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:53.594+00:00 2026-10-15T04:31:53Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:53.594+00:00 2026-10-15T04:31:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:54.612+00:00 2026-10-15T04:31:54Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:54.612+00:00 2026-10-15T04:31:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:55.605+00:00 2026-10-15T04:31:55Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:55.605+00:00 2026-10-15T04:31:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T04:31:56.610+00:00 2026-10-15T04:31:56Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T04:31:56.610+00:00 2026-10-15T04:31:56Z open_rl:trainer_ready:Qwen/Qwen3-8B diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log new file mode 100644 index 00000000..f36aa39c --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T04:28:48.737+00:00 2026-10-15T04:28:48.736460900Z {"level":"info","ts":"2026-10-15T04:28:48Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"9810872f-787e-462c-b16a-dceac5ec5ffa","claim":"claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} +2026-10-15T04:29:47.974+00:00 2026-10-15T04:29:47.973626989Z {"level":"info","ts":"2026-10-15T04:29:47Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"a0e0e96a-42aa-4961-903d-5120a533b95f","claim":"claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json new file mode 100644 index 00000000..90585ed3 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json @@ -0,0 +1,267 @@ +{ + "case": "D-8b-h100-cached", + "collected": "2026-10-15T04:32:14.200+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 04:28:47.756000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen3-8B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 04:28:47.756000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 04:29:39.925000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "db28c229-9bd6-4845-bd41-edb2e3018375" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 04:29:42.710000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 04:29:45.623000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 04:29:47.732000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 04:29:47.733000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 04:29:49.654000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 04:29:49.657000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 04:31:50.315000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4\n\nQuestion: What is " + }, + "client.end": { + "ts": "2026-10-15 04:31:50.316000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen3-8b-0-sampler", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "pod": "orw-lora-qwen-qwen3-8b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 04:29:47+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 04:29:47.960000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 04:29:47.960000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 04:29:47.960000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 04:29:52+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 04:29:48+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 04:29:48+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 04:29:52+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 04:29:52+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 04:29:48+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 04:29:52+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 04:29:48+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 04:29:51+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "sampler.event_Created": { + "ts": "2026-10-15 04:29:51+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 04:29:52+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 04:29:59.120383+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 04:30:16.823040+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 04:30:17.937691+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:17 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 04:30:18.941839+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 04:30:31.464946+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:31 [default_loader.py:430] Loading weights took 12.53 seconds", + "self_reported_s": 12.53 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 04:30:32.103943+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:32 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.439416 seconds", + "self_reported_s": 14.439416 + }, + "sampler.compile_done": { + "ts": "2026-10-15 04:31:08.896773+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:08 [monitor.py:53] torch.compile took 35.76 s in total", + "self_reported_s": 35.76 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 04:31:29.315428+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:29 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", + "self_reported_s": 16.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 04:31:29.941703+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:29 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 04:31:48.438271+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:48 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.33 s", + "self_reported_s": 76.33 + }, + "sampler.ready_key": { + "ts": "2026-10-15 04:31:49+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" + }, + "trainer": { + "workload": "lora-qwen-qwen3-8b-0-trainer", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "pod": "orw-lora-qwen-qwen3-8b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 04:28:48+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 04:28:48.508000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 04:28:48.508000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 04:28:48.508000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 04:28:53+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 04:28:48+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 04:28:48+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 04:28:53+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 04:28:53+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 04:28:48+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 04:28:52+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 04:28:48+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 04:28:52+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "trainer.event_Created": { + "ts": "2026-10-15 04:28:52+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 04:28:52+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 04:28:57.210385+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 04:28:57.431621+00:00", + "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 04:29:37.342144+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 04:29:37.343864+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 04:29:37+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.md b/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.md new file mode 100644 index 00000000..f719712d --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.md @@ -0,0 +1,109 @@ +# Cold start timeline: D-8b-h100-cached + +t=0 is the client's `create_model` call at 2026-10-15T04:28:47.756+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen3-8b-0-trainer` on `gke-open-rl-dra-h100-trainer-702095b4-1g3a` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.workload_created | 04:28:48.000 | 0.2 | | Workload creationTimestamp (apiserver) | +| trainer.pod_created | 04:28:48.000 | 0.2 | | pod creationTimestamp (apiserver) | +| trainer.pod_Initialized | 04:28:48.000 | 0.2 | | pod condition Initialized | +| trainer.pod_PodScheduled | 04:28:48.000 | 0.2 | | pod condition PodScheduled | +| trainer.event_Scheduled | 04:28:48.000 | 0.2 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a | +| trainer.workload_first_seen_by_poller | 04:28:48.508 | 0.8 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 04:28:48.508 | 0.8 | | workloads.tsv poller: status.podName set | +| trainer.workload_reports_node | 04:28:48.508 | 0.8 | | workloads.tsv poller: status.nodeName set | +| trainer.container_started | 04:28:52.000 | 4.2 | | containerStatuses.state.running.startedAt | +| trainer.event_Pulled | 04:28:52.000 | 4.2 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" already present on machine and can | +| trainer.event_Created | 04:28:52.000 | 4.2 | | event: Container created | +| trainer.event_Started | 04:28:52.000 | 4.2 | | event: Container started | +| trainer.condition_Placed_True | 04:28:53.000 | 5.2 | | Workload condition Placed | +| trainer.pod_Ready | 04:28:53.000 | 5.2 | | pod condition Ready | +| trainer.pod_ContainersReady | 04:28:53.000 | 5.2 | | pod condition ContainersReady | +| trainer.process_first_log | 04:28:57.210 | 9.5 | | container log, first line | +| trainer.load_start | 04:28:57.431 | 9.7 | | container log: Loading base model Qwen/Qwen3-8B to cuda:0... | +| trainer.ready_key | 04:29:37.000 | 49.2 | | redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B | +| trainer.load_done | 04:29:37.342 | 49.6 | | container log: Successfully loaded. | +| trainer.processor_started | 04:29:37.343 | 49.6 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1). | + +## sampler: `orw-lora-qwen-qwen3-8b-0-sampler` on `gke-open-rl-dra-h100-sampler-a721f127-0853` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_created | 04:29:47.000 | 59.2 | | Workload creationTimestamp (apiserver) | +| sampler.workload_first_seen_by_poller | 04:29:47.960 | 60.2 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_reports_pod | 04:29:47.960 | 60.2 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 04:29:47.960 | 60.2 | | workloads.tsv poller: status.nodeName set | +| sampler.pod_created | 04:29:48.000 | 60.2 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 04:29:48.000 | 60.2 | | pod condition Initialized | +| sampler.pod_PodScheduled | 04:29:48.000 | 60.2 | | pod condition PodScheduled | +| sampler.event_Scheduled | 04:29:48.000 | 60.2 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853 | +| sampler.event_Pulled | 04:29:51.000 | 63.2 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" already present on machine and can | +| sampler.event_Created | 04:29:51.000 | 63.2 | | event: Container created | +| sampler.condition_Placed_True | 04:29:52.000 | 64.2 | | Workload condition Placed | +| sampler.pod_Ready | 04:29:52.000 | 64.2 | | pod condition Ready | +| sampler.pod_ContainersReady | 04:29:52.000 | 64.2 | | pod condition ContainersReady | +| sampler.container_started | 04:29:52.000 | 64.2 | | containerStatuses.state.running.startedAt | +| sampler.event_Started | 04:29:52.000 | 64.2 | | event: Container started | +| sampler.process_first_log | 04:29:59.120 | 71.4 | | container log, first line | +| sampler.engine_init_start | 04:30:16.823 | 89.1 | | container log: (EngineCore pid=169) INFO 10-15 04:30:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 04:30:17.937 | 90.2 | | container log: (EngineCore pid=169) INFO 10-15 04:30:17 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 04:30:18.941 | 91.2 | | container log: (EngineCore pid=169) INFO 10-15 04:30:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 04:30:31.464 | 103.7 | 12.53 | container log: (EngineCore pid=169) INFO 10-15 04:30:31 [default_loader.py:430] Loading weights took 12.53 seconds | +| sampler.model_loaded | 04:30:32.103 | 104.3 | 14.439416 | container log: (EngineCore pid=169) INFO 10-15 04:30:32 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.439416 seconds | +| sampler.compile_done | 04:31:08.896 | 141.1 | 35.76 | container log: (EngineCore pid=169) INFO 10-15 04:31:08 [monitor.py:53] torch.compile took 35.76 s in total | +| sampler.graphs_captured | 04:31:29.315 | 161.6 | 16.0 | container log: (EngineCore pid=169) INFO 10-15 04:31:29 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB | +| sampler.kv_cache_sized | 04:31:29.941 | 162.2 | | container log: (EngineCore pid=169) INFO 10-15 04:31:29 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB | +| sampler.engine_init_done | 04:31:48.438 | 180.7 | 76.33 | container log: (EngineCore pid=169) INFO 10-15 04:31:48 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.33 s | +| sampler.ready_key | 04:31:49.000 | 181.2 | | redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 04:28:47.756 | 0.0 | +| client.create_model.start | 04:28:47.756 | 0.0 | +| client.create_model.done | 04:29:39.925 | 52.2 | +| client.forward_backward.start | 04:29:42.710 | 55.0 | +| client.forward_backward.done | 04:29:45.623 | 57.9 | +| client.optim_step.done | 04:29:47.732 | 60.0 | +| client.save_weights_and_get_sampling_client.start | 04:29:47.733 | 60.0 | +| client.save_weights_and_get_sampling_client.done | 04:29:49.654 | 61.9 | +| client.sample.start | 04:29:49.657 | 61.9 | +| client.sample.done | 04:31:50.315 | 182.6 | +| client.end | 04:31:50.316 | 182.6 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.2 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.4 | +| weights load | trainer.load_start | trainer.load_done | 39.9 | +| ready key | trainer.load_done | trainer.ready_key | -0.3 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 52.2 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.3 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 3.0 | +| container create | sampler.event_Pulled | sampler.container_started | 1.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 7.1 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 17.7 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 13.5 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.4 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.5 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.6 | +| client: first sample blocked | client.sample.start | client.sample.done | 120.7 | + diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt new file mode 100644 index 00000000..777815ab --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T04:28:43.565+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt new file mode 100644 index 00000000..0d162c07 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json new file mode 100644 index 00000000..3f3cb447 --- /dev/null +++ b/runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:47Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038592335647008", + "uid": "aac5fdd5-1826-49c7-8392-5165024b0a69" + }, + "spec": { + "accelerator": { + "memory": "29Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "name": "worker", + "resources": { + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "64aee2d3-084c-46a4-9c77-df7f6442e580", + "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T04:29:52Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "29Gi", + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038533448655004", + "uid": "cde28d2b-7497-4559-ae79-2c350558ddd1" + }, + "spec": { + "accelerator": { + "memory": "20Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", + "name": "worker", + "resources": { + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "6521caef-db14-4d7b-8477-e644e69919ee", + "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T04:28:53Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "20Gi", + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/REPORT.md b/runs/2026-10-15_cold_start/REPORT.md new file mode 100644 index 00000000..ec0ed106 --- /dev/null +++ b/runs/2026-10-15_cold_start/REPORT.md @@ -0,0 +1,239 @@ +# Cold start on open-rl-dra, measured + +Date: 2026-10-15. Cluster `open-rl-dra` (GKE, us-central1), v0.0.6 bundle with dev +api-server and worker images. Demo definition, fixed: one LoRA job on a base model +already in the Hugging Face cache, client on a laptop behind a port-forward, trainer +and sampler on separate nodes. No node pool was recreated and no storage was added. + +This replaces the derived timing table in `docs/briefs/cold-start-options.md` +section 5 with measurements. Every number below comes from a primary record; +anything not measured says so. + +## 1. How it was measured + +- `scripts/cold_start_client.py` runs the demo and stamps each client phase with the + laptop clock. +- `scripts/cold_start_timeline.py watch|collect|timeline` records, per run: the + Workload and pod objects (apiserver timestamps), Kubernetes events (kubelet's + `Pulled ... in Xs` carries the pull it measured), the worker containers' logs with + containerd timestamps (vLLM and the trainer print their own phase timings), the + Redis ready keys polled once a second from inside the Redis pod, and the API + server and scheduler logs. It joins them into `/timeline.md`. +- `scripts/cold_start_case.sh` steers placement with the `openrl.io/` node + labels, clears the previous workers, and runs one case end to end. +- NFS client counters (`/proc/self/mountstats`) were sampled inside the worker pods + during the 8B loads: bytes, read ops, bytes per op, round-trip time, queue time. +- `scripts/cold_start_fio.sh` ran fio from an L4 node and an H100 node against real + safetensors files on the shared volume. +- `scripts/cold_start_monitoring.py` pulled Filestore's `read_bytes_count` and + `read_ops_count` from Cloud Monitoring for each window. +- `scripts/cold_start_resident.py` (mincore) checked page-cache residency of the + weights inside a pod. + +Clocks: apiserver, nodes, the Redis pod and the laptop all agree to within a second +(container `Started` event versus first log line, and `ready_key` versus the +worker's own "loaded" line, differ by under 1 s in every run). + +## 2. Before the fix: four cases, one run each + +Worker image `open-rl-server:agent3-9f8dc13`. "Image cached" means the tag was on the +node; "cold" means it was not. Seconds. + +| Case | Nodes | Total to first sample | create_model blocked | Trainer image pull | Trainer weights load | First sample blocked | Sampler image pull | vLLM weights load | torch.compile | profile + graph capture | +|---|---|---|---|---|---|---|---|---|---|---| +| B: Qwen2.5-0.5B, L4, image cached | sr2c / bs96 | **132.0** | 17.6 | 0 (present) | 4.2 | 106.7 | 0 (present) | 3.0 | 31.4 | 26.7 | +| A: Qwen2.5-0.5B, L4, image cold | lvw8 / 2208 | **471.0** | 186.7 | 168.6 | 4.3 | 277.0 | 168.6 | 3.3 | 31.5 | 26.6 | +| C: Qwen3-8B, H100, image cold | 1g3a / 0853 | **1350.2** | 1008.6 | 92.5 | **903.5** | 331.4 | 95.3 | 103.6 | 36.9 | 39.3 | +| D: Qwen3-8B, H100, image cached, rerun of the same nodes | 1g3a / 0853 | **182.6** | 52.2 | 0 (present) | 39.9 | 120.7 | 0 (present) | 13.5 | 37.4 | 39.5 | + +Fixed costs seen in every run: placement (Workload to scheduled pod) 0.1 to 1.4 s; +container create 0 to 1 s; trainer process start to load start 5 to 7 s; sampler +process start to first log 7 to 12 s, then 18 to 28 s of imports until vLLM's engine +init line; ready key within 1.5 s of the worker's own "loaded" line. + +Sources: `/timeline.md`. Image pull durations are kubelet's own, from the +`Pulled` event message. `D-8b-h100-cached-interrupted/` is an earlier attempt at D +whose trainer load (266 s) is valid but whose client was stranded when the laptop's +port-forward dropped for 40 minutes and the reaper released the workers; the +numbers above are from the clean rerun. + +### 2.1 The largest phase per case + +- **0.5B on L4 (A and B):** the sampler's vLLM start: torch.compile (31 s) plus + profiling and CUDA graph capture (27 s), 58 s of the 107 s from sampler pod to + first sample, with another 39 s of process start and imports before it. The + weights are 3 s. On a cold node the two image pulls (169 s each) dwarf everything. + The brief's fix for the compile and capture is A3 and A4; not implemented in this + round (see section 7). +- **8B on H100 (C):** the trainer's weights load, 903 s of 1350. The brief's fix for + this phase is Group B (new storage), excluded this round. Section 3 shows why a + code change removes most of it instead. + +## 3. Why the trainer takes 15 minutes to read what vLLM reads in 100 seconds + +Both workers in case C read the same 16 GB from the same Filestore Basic HDD +instance, on nodes with equally cold page caches (both nodes were created the day +before and had never run this model). From `/proc/self/mountstats` inside each pod: + +| Reader | Wall | Bytes from server | NFS READ ops | Bytes per op | Mean RTT per op | Ops in flight (RTT x rate) | Throughput | +|---|---|---|---|---|---|---|---| +| Trainer, `transformers.from_pretrained` (mmap) | 903 s | 16.4 GB | 63,031 | 260 KiB | 27.9 ms | about 2 | 10.8 MiB/s (60 s sample: 43.5 ops/s, 46 ms RTT, 0 ms queue, CPU 4 millicores) | +| Sampler, vLLM 0.30 loader | 102.5 s | 16.4 GB | 19,591 | 846 KiB | 11.0 ms | about 2 | 153 MiB/s | + +The trainer's reads are page faults on a memory-mapped file: one request of +roughly the NFS readahead window at a time, each waiting a full disk round trip, +with the CPU idle. vLLM 0.30 detects the network filesystem and first streams the +files into the page cache with 8 threads and 16 MiB reads (its log: +`Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB`, then +`Prefetching checkpoint files into page cache started (in background, +num_threads=8, block_size=16777216 bytes)`), so its mmap afterwards hits memory. +The difference is concurrency and request size, not the storage. + +Two more facts the before-runs established: + +- **The node's page cache does not survive the pod.** Inside the case C pods, mincore + showed all 15.3 GiB resident. The next trainer pod on the same node (case D, 50 s + later) read all 16.4 GB from the server again (mountstats: 15.06 GB after 167 s). + Filestore is mounted per pod by the CSI driver and unmounted when the last pod on + the node releases it, which drops the cache. +- **Filestore's server-side cache does matter.** The per-read round trip for the + same files fell from 27.9 ms (case C, first read in days) to 5.2 ms (first attempt + at D) to 1.15 ms (clean D rerun), and the trainer's serialized load fell from + 903 s to 266 s to 40 s with no change in its read pattern. A demo repeated on a + warm instance is a different measurement from the first run of the day. + +## 4. The Filestore hypothesis, tested directly + +fio from the nodes, O_DIRECT so the node's page cache does not answer +(`fio-*/summary.md`). + +Against a file read minutes earlier (hot in the server cache), from L4 node lvw8: + +| Job | MiB/s | IOPS | Mean latency | +|---|---|---|---| +| sequential 1 MiB, one stream | 478.7 | 479 | 2.09 ms | +| random 4 KiB, one stream | 4.2 | 1,066 | 0.94 ms | +| sequential 256 KiB, queue depth 1 (the trainer's pattern) | 210.2 | 841 | 1.19 ms | +| sequential 1 MiB, four streams | 1,775.4 | 1,775 | 2.25 ms | + +Against a file not read that day (cold on disk), from H100 node 1g3a: + +| Job | MiB/s | IOPS | Mean latency | +|---|---|---|---| +| sequential 1 MiB, one stream | 16.2 | 16 | 61.7 ms | +| random 4 KiB, one stream | 0.1 | 33 | 30.2 ms | +| sequential 256 KiB, queue depth 1 | 44.8 (partly cached by the previous job) | 179 | 5.6 ms | +| sequential 1 MiB, four streams | 134.7 | 135 | 29.7 ms | + +Answer to the brief's open question: the weights load is **latency-bound, not +bandwidth-bound and not IOPS-bound**. One stream against cold disk gets 16 MiB/s +because each request waits 30 to 60 ms; four streams get 135 MiB/s; a hot server +cache gives 480 MiB/s on one stream and 1.7 GiB/s on four. The "100 MiB/s" +Basic HDD figure in the brief was never the binding constraint in any run. + +**Cloud Monitoring cannot confirm client throughput.** `file.googleapis.com/nfs/server/read_bytes_count` +showed about 10 MiB/s during the trainer's cold load (matches), 0 during the +sampler's 153 MiB/s load and during the 1.7 GiB/s fio run (both served from the +server cache), and 7 to 22 MiB/s during the cold fio run that measured 16 to +135 MiB/s. The metric appears to count disk-backed reads with coarse sampling. It is +evidence that cold disk reads happened, not a measure of what clients received. + +## 5. The fix + +`src/training/prefetch.py`: before `from_pretrained`, read every checkpoint file into +the page cache with 8 threads and 16 MiB reads, in 256 MiB tasks so large shards +spread across threads. Automatic on network filesystems when the files fit in 90% +of available memory, `OPEN_RL_PREFETCH=always|off` to force or disable, +`OPEN_RL_PREFETCH_THREADS` and `OPEN_RL_PREFETCH_BLOCK_MIB` to tune. Wired into the +LoRA, FFT and Automodel trainers. This is the mechanism vLLM already uses on the +sampler side, applied to the trainer. + +Also implemented, A6 from the brief: the trainer and sampler ready keys now carry a +JSON record (`ready_at`, `phases`: process start, prefetch result, load start and +end; engine build start and end for the sampler), and `GET /api/v1/openrl/models/{id}` +returns it per worker, so the next slow start can be attributed without logs. + +Images: `open-rl-server:coldstart1-dca985a`, `open-rl-api-server:coldstart1-dca985a` +(commit `053fb8d` on branch `feat/cold-start-measure`). Deployed by patching +ConfigMap `open-rl-images` and the api-server Deployment; the v0.0.6 bundle is +otherwise unchanged. The automodel image was not rebuilt. + +## 6. After the fix + +Worker and api-server images `coldstart1-dca985a`. Same node pairs, same procedure. +The first run on each pair pulls the new tag (cold image); the next runs find it +cached. By the time these ran, the Qwen3-8B and Qwen2.5-0.5B files were hot in +Filestore's server cache (section 3), so the before/after pairs below compare +against the warm-cache "before" where one exists, and the within-run control is +the sampler: vLLM's own 8-thread prefetch loads the same files in the same minute. + +| Run | Total to first sample | create_model blocked | Trainer weights load (prefetch + mmap) | First sample blocked | vLLM weights load | compile | capture | +|---|---|---|---|---|---|---|---| +| after-A1: 0.5B, L4, image cold | 493.7 | 197.1 | 15.0 (prefetch 12.4 s at 76 MiB/s, 4 streams: a 0.92 GiB file splits into four 256 MiB tasks; cold disk) | 289.2 | 2.4 | 31.5 | 26.7 | +| after-B1: 0.5B, L4, cached | 157.4 | 15.5 | 2.5 (prefetch 0.7 s, 1,341 MiB/s) | 134.6 (32 s of it between pod created and container started, not seen in the other runs) | 2.5 | 31.4 | 26.9 | +| after-B2: 0.5B, L4, cached | 130.5 | 15.8 | 2.5 | 107.4 | 2.6 | 31.3 | 26.8 | +| after-C1: 8B, H100, image cold | 329.3 | 111.5 (90 s of it the image pull) | **13.0** (prefetch 9.4 s, 1,666 MiB/s) | 207.9 (90 s pull) | 14.0 | 37.5 | 39.9 | +| after-D1: 8B, H100, cached | 154.5 | 25.3 | **13.5** | 119.4 | 12.9 | 37.6 | 39.3 | +| after-D2: 8B, H100, cached | 154.3 | 24.1 | **12.1** | 119.9 | 13.1 | 37.2 | 39.5 | +| after-E1: gemma-4-e2b 9.54 GiB, H100, trainer only, file cold on disk | n/a | 103.7 | **92.5** (prefetch 85.1 s at 115 MiB/s; 9,809 NFS reads of 1 MiB, mean RTT 119 ms, about 14 in flight) | n/a | n/a | n/a | n/a | + +Before and after, the phase the fix targets: + +| Comparison | Before | After | Server cache state | Factor | +|---|---|---|---|---| +| 8B trainer load, warm server cache | 39.9 s (D) | 12.1, 13.0, 13.5 s (median 13.0) | hot, both | 3.1x | +| 8B trainer load versus vLLM in the same run | 903.5 s vs 103.6 s (C, cold); 39.9 vs 13.5 (D, hot) | 13.0 vs 14.0 (C1); 13.5 vs 12.9 (D1); 12.1 vs 13.1 (D2) | | trainer now equals the 8-thread loader | +| Cold-disk read rate in the trainer | 10.8 MiB/s (60 s sample, case C) to 17 MiB/s (whole load) at 260 KiB per read, about 2 in flight | 115 MiB/s (E1) at 1 MiB per read, about 14 in flight | cold, both | 7 to 11x | +| 8B total to first sample, everything cached | 182.6 s | 154.3, 154.5 s | hot | 1.2x; the rest is vLLM compile + capture (77 s) and startup | +| 0.5B trainer load, hot | 4.2 s | 2.5 s | hot | the prefetch is 0.7 s; no regression | +| 0.5B trainer load, cold disk | 4.3 s (file was hot then) | 15.0 s (file cold, 4 streams) | not comparable | see note below | + +The 8B cold-disk before (903 s) could not be rerun after the fix because the files +stayed hot in the server cache; E1 on gemma-4-e2b, read from disk at 119 ms per +request, is the cold-disk measurement of the fixed path. Projected from E1's rate, +the case C trainer load would be about 135 s instead of 903 s; measured directly +only when the cache goes cold again, which this report does not claim. + +Not measured three times: the cold-image cases (A, C) ran once each after the fix, +because the only way to make a node forget the image again is to delete it from +containerd on the node, which this round did not do. The cached cases ran twice. + +**Where the targeted phase did not shrink:** the 0.5B cold-disk load went from 4 +to 15 s. The old path read a hot file in 4 s; the new path read a cold file with +four streams. The honest statement is that for a 0.92 GiB file the prefetch's +256 MiB task size leaves half the threads idle; a smaller task size for small +checkpoints is the obvious follow-up, not measured here. + +Side effect worth noting: the sampler's vLLM load fell from 103.6 s (C) to 14 s +(C1) between the two cold-image 8B runs. That is the server cache warming between +runs, not the fix; it shows how much the first demo of the day differs from the +second. + +## 7. What this round did not do, with the numbers to decide it + +- **A3/A4, the sampler's compile and capture:** 31 to 37 s of torch.compile and 27 + to 40 s of profiling plus CUDA graph capture on every sampler start, in all eight + runs. vLLM writes its compile cache to `/root/.cache/vllm` inside the container, + which dies with the pod. For the 0.5B demo this is the largest phase once the + image is on the node. +- **Image pulls:** 169 s per pull on L4 (pd-balanced boot disk), 92 to 95 s on H100 + (local SSD), measured by kubelet, every time a node lacks the tag; two pulls per + cold demo. A2 (pre-pull) and C1 (image streaming) in the brief. +- **Keeping the page cache across pods** needs the volume to stay mounted on the + node (a pod that holds it) or a node-local copy: B2 in the brief. +- **The reaper during a connectivity drop** ended one measurement run; the client was + never told (issues #246, #247; design 016 phase 2). + +## 8. What confirms this independently of this work + +- kubelet's own pull durations: `kubectl -n openrl-system get events --field-selector reason=Pulled -o custom-columns='T:.firstTimestamp,P:.involvedObject.name,M:.message'`, + or after the pods are gone, `gcloud logging read 'logName="projects/cdrollouts-sunilarora/logs/events" AND jsonPayload.reason="Pulled" AND jsonPayload.involvedObject.name:"orw-"' --freshness 1d --format='value(timestamp,jsonPayload.message)'`. +- vLLM's own lines in the sampler log: `kubectl -n openrl-system logs | grep -E 'Loading weights took|torch.compile took|Graph capturing finished|init engine'`; + the trainer's `[prefetch] done:` and `Successfully loaded.` lines with `--timestamps`. +- The NFS client counters, which neither OpenRL nor vLLM write: `kubectl -n openrl-system exec -- grep -A40 /mnt/shared /proc/self/mountstats` + (`bytes:` field 5 is bytes read from the server; `READ:` fields are ops, ..., total RTT ms). +- Cloud Monitoring, for disk-backed reads only: `python3 scripts/cold_start_monitoring.py --start ... --end ... --out /tmp/m`. +- A stopwatch rehearsal of the demo by someone else, from a fresh terminal: + `uv --project examples run python scripts/cold_start_client.py base_model=Qwen/Qwen3-8B out=/tmp/rehearsal.jsonl`, + reading `t` in the printed lines; run it once on a node pair that has not run the + model that day and once again two minutes later. diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log new file mode 100644 index 00000000..f8ffab7c --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log @@ -0,0 +1,35 @@ +watching; recorders [12362, 12363, 12365, 12367, 12369, 12364, 12366, 12368] write into runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold +collected 2 worker log(s) into runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.9 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 172.0 | +| container create | trainer.event_Pulled | trainer.container_started | 1.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.2 | +| weights load | trainer.load_start | trainer.load_done | 15.0 | +| ready key | trainer.load_done | trainer.ready_key | 0.7 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 197.1 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.1 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 186.0 | +| container create | sampler.event_Pulled | sampler.container_started | 1.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 23.2 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 15.2 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.7 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.4 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.5 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.7 | +| ready key | sampler.engine_init_done | sampler.ready_key | 1.2 | +| client: first sample blocked | client.sample.start | client.sample.done | 289.2 | + diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log new file mode 100644 index 00000000..72baa052 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log @@ -0,0 +1,75 @@ +2026-10-15T04:47:50.759+00:00 2026-10-15T04:47:50.747267827Z INFO: 10.32.3.1:50030 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:47:51.373+00:00 2026-10-15T04:47:51.375072109Z INFO: 127.0.0.1:58948 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T04:47:51.625+00:00 2026-10-15T04:47:51.626679728Z INFO: 127.0.0.1:58962 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T04:47:51.706+00:00 2026-10-15T04:47:51.698589265Z INFO: 127.0.0.1:58962 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T04:47:51.964+00:00 2026-10-15T04:47:51.960105348Z [API_SERVER] enqueued op=create_model request_id=c34fd223-1dd8-443e-850e-68d16b0d897e model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:47:51.965+00:00 2026-10-15T04:47:51.961209781Z INFO: 127.0.0.1:58976 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T04:48:00.748+00:00 2026-10-15T04:48:00.747101104Z INFO: 10.32.3.1:44106 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:48:01.881+00:00 2026-10-15T04:48:01.882432111Z INFO: 127.0.0.1:39078 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:48:10.748+00:00 2026-10-15T04:48:10.746655455Z INFO: 10.32.3.1:43406 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:48:20.762+00:00 2026-10-15T04:48:20.747630744Z INFO: 10.32.3.1:50344 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:48:30.750+00:00 2026-10-15T04:48:30.747452523Z INFO: 10.32.3.1:38480 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:48:40.748+00:00 2026-10-15T04:48:40.747887168Z INFO: 10.32.3.1:43688 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:48:42.107+00:00 2026-10-15T04:48:42.108647369Z INFO: 127.0.0.1:44312 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:48:50.747+00:00 2026-10-15T04:48:50.746753362Z INFO: 10.32.3.1:56992 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:49:00.759+00:00 2026-10-15T04:49:00.748474238Z INFO: 10.32.3.1:51752 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:49:10.753+00:00 2026-10-15T04:49:10.747254442Z INFO: 10.32.3.1:60438 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:49:20.748+00:00 2026-10-15T04:49:20.747384062Z INFO: 10.32.3.1:55404 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:49:30.744+00:00 2026-10-15T04:49:30.748502393Z INFO: 10.32.3.1:51894 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:49:32.332+00:00 2026-10-15T04:49:32.338576070Z INFO: 127.0.0.1:50908 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:49:40.737+00:00 2026-10-15T04:49:40.746618093Z INFO: 10.32.3.1:59074 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:49:50.742+00:00 2026-10-15T04:49:50.747114846Z INFO: 10.32.3.1:41452 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:50:00.737+00:00 2026-10-15T04:50:00.747761065Z INFO: 10.32.3.1:58260 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:50:10.736+00:00 2026-10-15T04:50:10.747653348Z INFO: 10.32.3.1:50724 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:50:12.511+00:00 2026-10-15T04:50:12.524698371Z INFO: 127.0.0.1:57438 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:50:20.733+00:00 2026-10-15T04:50:20.746488844Z INFO: 10.32.3.1:37876 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:50:30.735+00:00 2026-10-15T04:50:30.747506581Z INFO: 10.32.3.1:46886 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:50:40.734+00:00 2026-10-15T04:50:40.747651246Z INFO: 10.32.3.1:47538 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:50:50.734+00:00 2026-10-15T04:50:50.747642546Z INFO: 10.32.3.1:42666 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:51:00.735+00:00 2026-10-15T04:51:00.747457778Z INFO: 10.32.3.1:51346 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:51:02.734+00:00 2026-10-15T04:51:02.748492339Z INFO: 127.0.0.1:33774 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:51:08.372+00:00 2026-10-15T04:51:08.384894234Z INFO: 127.0.0.1:33804 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T04:51:10.734+00:00 2026-10-15T04:51:10.747373366Z INFO: 10.32.3.1:44790 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:51:11.018+00:00 2026-10-15T04:51:11.033251000Z [API_SERVER] enqueued op=forward_backward request_id=82deca49-4ee4-496f-825c-c12a291fb417 model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:51:11.018+00:00 2026-10-15T04:51:11.033591274Z INFO: 127.0.0.1:33804 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T04:51:14.259+00:00 2026-10-15T04:51:14.268225387Z [API_SERVER] enqueued op=optim_step request_id=f6a0aff8-ac17-496d-a35f-82a1866beb3c model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:51:14.259+00:00 2026-10-15T04:51:14.269572118Z INFO: 127.0.0.1:33804 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T04:51:15.185+00:00 2026-10-15T04:51:15.196153109Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=29042980-72d1-47bc-b560-caa782c86947 model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:51:15.185+00:00 2026-10-15T04:51:15.196730876Z INFO: 127.0.0.1:33804 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T04:51:15.847+00:00 2026-10-15T04:51:15.861994037Z INFO: 127.0.0.1:50962 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T04:51:20.733+00:00 2026-10-15T04:51:20.747509561Z INFO: 10.32.3.1:36522 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:51:30.734+00:00 2026-10-15T04:51:30.747769712Z INFO: 10.32.3.1:49072 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:51:40.735+00:00 2026-10-15T04:51:40.747089500Z INFO: 10.32.3.1:42936 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:51:50.807+00:00 2026-10-15T04:51:50.747657379Z INFO: 10.32.3.1:59096 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:52:00.735+00:00 2026-10-15T04:52:00.746837023Z INFO: 10.32.3.1:44932 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:52:02.930+00:00 2026-10-15T04:52:02.943641729Z INFO: 127.0.0.1:60640 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:52:10.736+00:00 2026-10-15T04:52:10.747747798Z INFO: 10.32.3.1:44974 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:52:20.736+00:00 2026-10-15T04:52:20.746678567Z INFO: 10.32.3.1:35664 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:52:30.734+00:00 2026-10-15T04:52:30.747126329Z INFO: 10.32.3.1:34208 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:52:40.735+00:00 2026-10-15T04:52:40.747196297Z INFO: 10.32.3.1:44174 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:52:50.734+00:00 2026-10-15T04:52:50.747115733Z INFO: 10.32.3.1:54892 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:52:51.758+00:00 2026-10-15T04:52:51.774099202Z INFO: 127.0.0.1:40930 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T04:52:53.102+00:00 2026-10-15T04:52:53.117722316Z INFO: 127.0.0.1:54688 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:53:00.733+00:00 2026-10-15T04:53:00.747018328Z INFO: 10.32.3.1:34592 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:53:10.739+00:00 2026-10-15T04:53:10.747952567Z INFO: 10.32.3.1:34466 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:53:20.735+00:00 2026-10-15T04:53:20.747402168Z INFO: 10.32.3.1:43034 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:53:30.735+00:00 2026-10-15T04:53:30.747874323Z INFO: 10.32.3.1:44336 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:53:40.734+00:00 2026-10-15T04:53:40.747391997Z INFO: 10.32.3.1:50648 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:53:43.288+00:00 2026-10-15T04:53:43.301447514Z INFO: 127.0.0.1:35782 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:53:50.735+00:00 2026-10-15T04:53:50.748058844Z INFO: 10.32.3.1:44446 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:54:00.741+00:00 2026-10-15T04:54:00.747477817Z INFO: 10.32.3.1:60436 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:54:10.733+00:00 2026-10-15T04:54:10.747726850Z INFO: 10.32.3.1:59106 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:54:20.768+00:00 2026-10-15T04:54:20.747514561Z INFO: 10.32.3.1:37768 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:54:23.560+00:00 2026-10-15T04:54:23.574498792Z INFO: 127.0.0.1:42366 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:54:30.734+00:00 2026-10-15T04:54:30.747280852Z INFO: 10.32.3.1:47062 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:54:40.733+00:00 2026-10-15T04:54:40.746421659Z INFO: 10.32.3.1:53944 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:54:50.739+00:00 2026-10-15T04:54:50.747856422Z INFO: 10.32.3.1:50702 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:55:00.789+00:00 2026-10-15T04:55:00.747454361Z INFO: 10.32.3.1:48938 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:55:10.733+00:00 2026-10-15T04:55:10.747058641Z INFO: 10.32.3.1:48650 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:55:20.734+00:00 2026-10-15T04:55:20.747467882Z INFO: 10.32.3.1:54104 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:55:23.758+00:00 2026-10-15T04:55:23.773354564Z INFO: 127.0.0.1:59436 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:55:30.733+00:00 2026-10-15T04:55:30.747006683Z INFO: 10.32.3.1:51618 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:55:40.733+00:00 2026-10-15T04:55:40.747561980Z INFO: 10.32.3.1:60968 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:55:50.733+00:00 2026-10-15T04:55:50.747513210Z INFO: 10.32.3.1:41264 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:56:00.733+00:00 2026-10-15T04:56:00.747491999Z INFO: 10.32.3.1:32776 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:56:10.733+00:00 2026-10-15T04:56:10.746647279Z INFO: 10.32.3.1:51500 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl new file mode 100644 index 00000000..b94dfe70 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T04:47:51.110+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T04:47:51.111+00:00", "t": 0.0, "phase": "create_model.start"} +{"ts": "2026-10-15T04:51:08.186+00:00", "t": 197.076, "phase": "create_model.done", "model_id": "c34fd223-1dd8-443e-850e-68d16b0d897e"} +{"ts": "2026-10-15T04:51:10.937+00:00", "t": 199.827, "phase": "forward_backward.start"} +{"ts": "2026-10-15T04:51:14.170+00:00", "t": 203.059, "phase": "forward_backward.done"} +{"ts": "2026-10-15T04:51:15.072+00:00", "t": 203.961, "phase": "optim_step.done"} +{"ts": "2026-10-15T04:51:15.072+00:00", "t": 203.962, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T04:51:15.661+00:00", "t": 204.551, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T04:51:15.662+00:00", "t": 204.551, "phase": "sample.start"} +{"ts": "2026-10-15T04:56:04.835+00:00", "t": 493.725, "phase": "sample.done", "text": " 4"} +{"ts": "2026-10-15T04:56:04.838+00:00", "t": 493.728, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log new file mode 100644 index 00000000..a1f626f6 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.000s create_model.start +[cold-start] 197.076s create_model.done {'model_id': 'c34fd223-1dd8-443e-850e-68d16b0d897e'} +[cold-start] 199.827s forward_backward.start +[cold-start] 203.059s forward_backward.done +[cold-start] 203.961s optim_step.done +[cold-start] 203.962s save_weights_and_get_sampling_client.start +[cold-start] 204.551s save_weights_and_get_sampling_client.done +[cold-start] 204.551s sample.start +[cold-start] 493.725s sample.done {'text': ' 4'} +[cold-start] 493.728s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt new file mode 100644 index 00000000..82fe3f4d --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T04:56:05.389+00:00 diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json new file mode 100644 index 00000000..7497a20b --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json @@ -0,0 +1,1311 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:10Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:10Z", + "name": "cold-start-fio.18de9a0ef295773d", + "namespace": "openrl-system", + "resourceVersion": "1792039090760383011", + "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5be63862", + "namespace": "openrl-system", + "resourceVersion": "1792039092515007000", + "uid": "355b59b9-3661-499d-8640-3f40c3c89375" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5ecfa51a", + "namespace": "openrl-system", + "resourceVersion": "1792039092561663021", + "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f61ddd1e4", + "namespace": "openrl-system", + "resourceVersion": "1792039092605935011", + "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:40:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:40:41Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:40:41Z", + "name": "cold-start-fio.18de9a32081cf908", + "namespace": "openrl-system", + "resourceVersion": "1792039241446287009", + "uid": "14a62e42-13ba-4526-a04d-a934702e1163" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:18Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:18Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:18Z", + "name": "cold-start-fio.18de9a3a9867ce9f", + "namespace": "openrl-system", + "resourceVersion": "1792039278208207001", + "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad2bcac91", + "namespace": "openrl-system", + "resourceVersion": "1792039279188751014", + "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad378cd74", + "namespace": "openrl-system", + "resourceVersion": "1792039279219679006", + "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad5aed221", + "namespace": "openrl-system", + "resourceVersion": "1792039279235135013", + "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:11Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:11Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:11Z", + "name": "cold-start-fio.18de9a70cd2c6a68", + "namespace": "openrl-system", + "resourceVersion": "1792039511048479001", + "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "resourceVersion": "1791941114537823021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Stopping container api-server", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", + "namespace": "openrl-system", + "resourceVersion": "1792039545418735013", + "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "namespace": "openrl-system", + "resourceVersion": "1792039545300351007", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", + "namespace": "openrl-system", + "resourceVersion": "1792039545387567001", + "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" + }, + "reason": "SuccessfulDelete", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546048591003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", + "namespace": "openrl-system", + "resourceVersion": "1792039546102351012", + "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:49Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:49Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", + "namespace": "openrl-system", + "resourceVersion": "1792039549712335019", + "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", + "namespace": "openrl-system", + "resourceVersion": "1792039558721215017", + "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", + "namespace": "openrl-system", + "resourceVersion": "1792039558783199013", + "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", + "namespace": "openrl-system", + "resourceVersion": "1792039558862303009", + "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "namespace": "openrl-system", + "resourceVersion": "1792039545940671018", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", + "namespace": "openrl-system", + "resourceVersion": "1792039546068975009", + "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" + }, + "reason": "SuccessfulCreate", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545232655018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78c9668cf7", + "namespace": "openrl-system", + "resourceVersion": "1792039545327119005", + "uid": "7b1433df-3524-478b-aba2-8979950e245e" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545464319018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78ef8b9e1b", + "namespace": "openrl-system", + "resourceVersion": "1792039545956079020", + "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:15Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875490623001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:15Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", + "namespace": "openrl-system", + "resourceVersion": "1792039875609279013", + "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:19Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:19Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", + "namespace": "openrl-system", + "resourceVersion": "1792039879126319008", + "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", + "namespace": "openrl-system", + "resourceVersion": "1792040061645247020", + "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", + "namespace": "openrl-system", + "resourceVersion": "1792040061849391002", + "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", + "namespace": "openrl-system", + "resourceVersion": "1792040062252079023", + "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:52Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672220015012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:52Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", + "namespace": "openrl-system", + "resourceVersion": "1792039672350591019", + "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:56Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:56Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:56Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", + "namespace": "openrl-system", + "resourceVersion": "1792039676166239015", + "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", + "namespace": "openrl-system", + "resourceVersion": "1792039844457167000", + "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", + "namespace": "openrl-system", + "resourceVersion": "1792039844632591021", + "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:45Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:45Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", + "namespace": "openrl-system", + "resourceVersion": "1792039845170959016", + "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", + "namespace": "openrl-system", + "resourceVersion": "1792038835732623016", + "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", + "namespace": "openrl-system", + "resourceVersion": "1792038835415695012", + "uid": "918abc9a-1f24-4767-8928-b202482f11ed" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt new file mode 100644 index 00000000..e82879b1 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt @@ -0,0 +1,120 @@ +2026-10-15T04:54:45.242588287Z INFO 10-15 04:54:45 [model.py:692] Resolved architecture: Qwen2ForCausalLM +2026-10-15T04:54:45.277423658Z INFO 10-15 04:54:45 [model.py:2030] Using max model len 8192 +2026-10-15T04:54:45.277455256Z INFO 10-15 04:54:45 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. +2026-10-15T04:54:45.327374906Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T04:54:45.587816185Z INFO 10-15 04:54:45 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T04:54:48.964429054Z WARNING 10-15 04:54:48 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T04:55:00.403987147Z (EngineCore pid=161) INFO 10-15 04:55:00 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T04:55:01.212591231Z (EngineCore pid=161) INFO 10-15 04:55:01 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_14e101098568499f8d634b8f3ba36063 backend=nccl +2026-10-15T04:55:01.219901527Z (EngineCore pid=161) INFO 10-15 04:55:01 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T04:55:01.219928991Z (EngineCore pid=161) INFO 10-15 04:55:01 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T04:55:02.093177635Z (EngineCore pid=161) INFO 10-15 04:55:02 [model_runner.py:396] Loading model from scratch... +2026-10-15T04:55:02.961631574Z (EngineCore pid=161) INFO 10-15 04:55:02 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T04:55:02.962148325Z (EngineCore pid=161) INFO 10-15 04:55:02 [flash_attn.py:1116] Using FlashAttention version 2 +2026-10-15T04:55:03.077952928Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T04:55:03.078372599Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T04:55:03.438996437Z (EngineCore pid=161) INFO 10-15 04:55:03 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.61 GiB. +2026-10-15T04:55:03.439031206Z (EngineCore pid=161) INFO 10-15 04:55:03 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T04:55:03.440364205Z (EngineCore pid=161) +Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T04:50:52.100407463Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T04:50:52.100446712Z +2026-10-15T04:50:52.221389461Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... +2026-10-15T04:50:52.374261114Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T04:50:53.898909828Z [prefetch] reading 1 file(s), 0.92 GiB, from nfs into the page cache with 8 threads +2026-10-15T04:51:06.274074801Z [prefetch] done: 0.92 GiB in 12.4 s (76 MiB/s) +2026-10-15T04:51:07.154043242Z +Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json new file mode 100644 index 00000000..7e1e5910 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "generateName": "open-rl-api-server-6d86b498f5-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "6d86b498f5", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + } + ], + "resourceVersion": "1792039570790271003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2ns79", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:59Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:45:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.11", + "podIPs": [ + { + "ip": "10.32.3.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:45:46Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "uid": "0752a694-3e75-44aa-9348-ae1c3f3fff13" + } + ], + "resourceVersion": "1792040062433135001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "ed5c4cfd-e94a-4a7f-a2a6-62ba293d8d04" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-7xf7q", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-7xf7q", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:54:22Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:51:15Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:54:22Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:54:22Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:51:15Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "25Gi" + }, + "containerID": "containerd://9e18e8280c01871067b298e040014667b0807f38eaf685466bc2c46ce2b4a65d", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:54:22Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-7xf7q", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.0.20", + "podIPs": [ + { + "ip": "10.32.0.20" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:51:15Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "uid": "4bef5a35-3bf8-4444-9043-47f86b502e3e" + } + ], + "resourceVersion": "1792039845726895012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "0772feaa-10d8-4243-ac6a-4b4ee0615042" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-hrg2h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-hrg2h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:50:45Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:47:52Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:50:45Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:50:45Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:47:52Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "21Gi" + }, + "containerID": "containerd://91dafb207ae62faa4acac97ecaac86ef915d83f785b30550632f947d515e7601", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:50:45Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-hrg2h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.13.26", + "podIPs": [ + { + "ip": "10.32.13.26" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:47:52Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log new file mode 100644 index 00000000..51432475 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log @@ -0,0 +1,309 @@ +2026-10-15T04:51:08.025+00:00 2026-10-15T04:51:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:09.029+00:00 2026-10-15T04:51:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:10.034+00:00 2026-10-15T04:51:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:11.039+00:00 2026-10-15T04:51:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:12.044+00:00 2026-10-15T04:51:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:13.052+00:00 2026-10-15T04:51:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:14.055+00:00 2026-10-15T04:51:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:15.071+00:00 2026-10-15T04:51:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:16.065+00:00 2026-10-15T04:51:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:17.071+00:00 2026-10-15T04:51:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:18.081+00:00 2026-10-15T04:51:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:19.081+00:00 2026-10-15T04:51:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:20.088+00:00 2026-10-15T04:51:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:21.105+00:00 2026-10-15T04:51:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:22.098+00:00 2026-10-15T04:51:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:23.103+00:00 2026-10-15T04:51:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:24.108+00:00 2026-10-15T04:51:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:25.113+00:00 2026-10-15T04:51:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:26.119+00:00 2026-10-15T04:51:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:27.123+00:00 2026-10-15T04:51:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:28.131+00:00 2026-10-15T04:51:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:29.133+00:00 2026-10-15T04:51:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:30.139+00:00 2026-10-15T04:51:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:31.144+00:00 2026-10-15T04:51:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:32.150+00:00 2026-10-15T04:51:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:33.155+00:00 2026-10-15T04:51:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:34.159+00:00 2026-10-15T04:51:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:35.164+00:00 2026-10-15T04:51:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:36.175+00:00 2026-10-15T04:51:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:37.175+00:00 2026-10-15T04:51:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:38.179+00:00 2026-10-15T04:51:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:39.184+00:00 2026-10-15T04:51:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:40.189+00:00 2026-10-15T04:51:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:41.313+00:00 2026-10-15T04:51:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:42.202+00:00 2026-10-15T04:51:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:43.204+00:00 2026-10-15T04:51:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:44.209+00:00 2026-10-15T04:51:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:45.227+00:00 2026-10-15T04:51:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:46.219+00:00 2026-10-15T04:51:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:47.224+00:00 2026-10-15T04:51:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:48.229+00:00 2026-10-15T04:51:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:49.234+00:00 2026-10-15T04:51:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:50.240+00:00 2026-10-15T04:51:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:51.348+00:00 2026-10-15T04:51:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:52.251+00:00 2026-10-15T04:51:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:53.255+00:00 2026-10-15T04:51:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:54.261+00:00 2026-10-15T04:51:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:55.266+00:00 2026-10-15T04:51:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:56.273+00:00 2026-10-15T04:51:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:57.278+00:00 2026-10-15T04:51:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:58.284+00:00 2026-10-15T04:51:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:51:59.351+00:00 2026-10-15T04:51:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:00.293+00:00 2026-10-15T04:52:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:01.309+00:00 2026-10-15T04:52:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:02.312+00:00 2026-10-15T04:52:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:03.310+00:00 2026-10-15T04:52:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:04.313+00:00 2026-10-15T04:52:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:05.319+00:00 2026-10-15T04:52:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:06.323+00:00 2026-10-15T04:52:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:07.328+00:00 2026-10-15T04:52:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:08.333+00:00 2026-10-15T04:52:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:09.338+00:00 2026-10-15T04:52:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:10.342+00:00 2026-10-15T04:52:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:11.347+00:00 2026-10-15T04:52:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:12.352+00:00 2026-10-15T04:52:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:13.358+00:00 2026-10-15T04:52:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:14.362+00:00 2026-10-15T04:52:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:15.421+00:00 2026-10-15T04:52:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:16.374+00:00 2026-10-15T04:52:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:17.420+00:00 2026-10-15T04:52:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:18.381+00:00 2026-10-15T04:52:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:19.386+00:00 2026-10-15T04:52:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:20.391+00:00 2026-10-15T04:52:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:21.397+00:00 2026-10-15T04:52:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:22.416+00:00 2026-10-15T04:52:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:23.406+00:00 2026-10-15T04:52:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:24.412+00:00 2026-10-15T04:52:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:25.416+00:00 2026-10-15T04:52:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:26.422+00:00 2026-10-15T04:52:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:27.609+00:00 2026-10-15T04:52:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:28.431+00:00 2026-10-15T04:52:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:29.442+00:00 2026-10-15T04:52:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:30.443+00:00 2026-10-15T04:52:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:31.449+00:00 2026-10-15T04:52:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:32.454+00:00 2026-10-15T04:52:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:33.459+00:00 2026-10-15T04:52:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:34.465+00:00 2026-10-15T04:52:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:35.472+00:00 2026-10-15T04:52:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:36.474+00:00 2026-10-15T04:52:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:37.479+00:00 2026-10-15T04:52:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:38.484+00:00 2026-10-15T04:52:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:39.501+00:00 2026-10-15T04:52:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:40.495+00:00 2026-10-15T04:52:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:41.502+00:00 2026-10-15T04:52:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:42.508+00:00 2026-10-15T04:52:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:43.510+00:00 2026-10-15T04:52:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:44.515+00:00 2026-10-15T04:52:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:45.520+00:00 2026-10-15T04:52:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:46.537+00:00 2026-10-15T04:52:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:47.531+00:00 2026-10-15T04:52:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:48.536+00:00 2026-10-15T04:52:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:49.541+00:00 2026-10-15T04:52:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:50.546+00:00 2026-10-15T04:52:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:51.551+00:00 2026-10-15T04:52:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:52.561+00:00 2026-10-15T04:52:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:53.561+00:00 2026-10-15T04:52:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:54.566+00:00 2026-10-15T04:52:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:55.570+00:00 2026-10-15T04:52:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:56.577+00:00 2026-10-15T04:52:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:57.581+00:00 2026-10-15T04:52:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:58.585+00:00 2026-10-15T04:52:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:52:59.590+00:00 2026-10-15T04:52:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:00.599+00:00 2026-10-15T04:53:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:01.601+00:00 2026-10-15T04:53:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:02.606+00:00 2026-10-15T04:53:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:03.610+00:00 2026-10-15T04:53:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:04.616+00:00 2026-10-15T04:53:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:05.622+00:00 2026-10-15T04:53:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:06.626+00:00 2026-10-15T04:53:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:07.631+00:00 2026-10-15T04:53:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:08.636+00:00 2026-10-15T04:53:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:09.643+00:00 2026-10-15T04:53:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:10.648+00:00 2026-10-15T04:53:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:11.652+00:00 2026-10-15T04:53:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:12.657+00:00 2026-10-15T04:53:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:13.663+00:00 2026-10-15T04:53:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:14.667+00:00 2026-10-15T04:53:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:15.672+00:00 2026-10-15T04:53:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:16.678+00:00 2026-10-15T04:53:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:17.683+00:00 2026-10-15T04:53:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:18.686+00:00 2026-10-15T04:53:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:19.691+00:00 2026-10-15T04:53:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:20.697+00:00 2026-10-15T04:53:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:21.702+00:00 2026-10-15T04:53:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:22.707+00:00 2026-10-15T04:53:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:23.713+00:00 2026-10-15T04:53:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:24.717+00:00 2026-10-15T04:53:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:25.722+00:00 2026-10-15T04:53:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:26.727+00:00 2026-10-15T04:53:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:27.733+00:00 2026-10-15T04:53:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:28.743+00:00 2026-10-15T04:53:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:29.743+00:00 2026-10-15T04:53:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:30.748+00:00 2026-10-15T04:53:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:31.760+00:00 2026-10-15T04:53:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:32.758+00:00 2026-10-15T04:53:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:33.764+00:00 2026-10-15T04:53:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:34.770+00:00 2026-10-15T04:53:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:35.779+00:00 2026-10-15T04:53:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:36.780+00:00 2026-10-15T04:53:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:37.786+00:00 2026-10-15T04:53:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:38.791+00:00 2026-10-15T04:53:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:39.797+00:00 2026-10-15T04:53:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:40.801+00:00 2026-10-15T04:53:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:41.807+00:00 2026-10-15T04:53:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:42.811+00:00 2026-10-15T04:53:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:43.816+00:00 2026-10-15T04:53:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:44.821+00:00 2026-10-15T04:53:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:45.826+00:00 2026-10-15T04:53:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:46.830+00:00 2026-10-15T04:53:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:47.837+00:00 2026-10-15T04:53:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:48.845+00:00 2026-10-15T04:53:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:49.847+00:00 2026-10-15T04:53:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:50.854+00:00 2026-10-15T04:53:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:51.856+00:00 2026-10-15T04:53:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:52.862+00:00 2026-10-15T04:53:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:53.867+00:00 2026-10-15T04:53:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:54.871+00:00 2026-10-15T04:53:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:55.877+00:00 2026-10-15T04:53:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:56.881+00:00 2026-10-15T04:53:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:57.886+00:00 2026-10-15T04:53:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:58.891+00:00 2026-10-15T04:53:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:53:59.898+00:00 2026-10-15T04:53:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:00.901+00:00 2026-10-15T04:54:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:01.907+00:00 2026-10-15T04:54:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:02.910+00:00 2026-10-15T04:54:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:03.923+00:00 2026-10-15T04:54:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:04.928+00:00 2026-10-15T04:54:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:05.930+00:00 2026-10-15T04:54:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:07.036+00:00 2026-10-15T04:54:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:07.939+00:00 2026-10-15T04:54:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:08.945+00:00 2026-10-15T04:54:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:09.950+00:00 2026-10-15T04:54:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:10.955+00:00 2026-10-15T04:54:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:11.989+00:00 2026-10-15T04:54:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:12.973+00:00 2026-10-15T04:54:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:13.973+00:00 2026-10-15T04:54:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:14.976+00:00 2026-10-15T04:54:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:15.981+00:00 2026-10-15T04:54:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:16.987+00:00 2026-10-15T04:54:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:17.991+00:00 2026-10-15T04:54:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:19.054+00:00 2026-10-15T04:54:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:20.002+00:00 2026-10-15T04:54:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:21.008+00:00 2026-10-15T04:54:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:22.074+00:00 2026-10-15T04:54:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:23.040+00:00 2026-10-15T04:54:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:24.021+00:00 2026-10-15T04:54:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:25.025+00:00 2026-10-15T04:54:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:26.029+00:00 2026-10-15T04:54:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:27.035+00:00 2026-10-15T04:54:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:28.050+00:00 2026-10-15T04:54:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:29.096+00:00 2026-10-15T04:54:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:30.056+00:00 2026-10-15T04:54:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:31.058+00:00 2026-10-15T04:54:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:32.065+00:00 2026-10-15T04:54:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:33.067+00:00 2026-10-15T04:54:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:34.072+00:00 2026-10-15T04:54:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:35.077+00:00 2026-10-15T04:54:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:36.085+00:00 2026-10-15T04:54:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:37.089+00:00 2026-10-15T04:54:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:38.093+00:00 2026-10-15T04:54:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:39.099+00:00 2026-10-15T04:54:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:40.106+00:00 2026-10-15T04:54:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:41.110+00:00 2026-10-15T04:54:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:42.114+00:00 2026-10-15T04:54:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:43.119+00:00 2026-10-15T04:54:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:44.202+00:00 2026-10-15T04:54:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:45.129+00:00 2026-10-15T04:54:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:46.133+00:00 2026-10-15T04:54:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:47.138+00:00 2026-10-15T04:54:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:48.142+00:00 2026-10-15T04:54:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:49.148+00:00 2026-10-15T04:54:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:50.152+00:00 2026-10-15T04:54:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:51.162+00:00 2026-10-15T04:54:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:52.163+00:00 2026-10-15T04:54:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:53.174+00:00 2026-10-15T04:54:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:54.172+00:00 2026-10-15T04:54:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:55.190+00:00 2026-10-15T04:54:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:56.182+00:00 2026-10-15T04:54:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:57.187+00:00 2026-10-15T04:54:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:58.192+00:00 2026-10-15T04:54:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:54:59.196+00:00 2026-10-15T04:54:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:00.202+00:00 2026-10-15T04:55:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:01.207+00:00 2026-10-15T04:55:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:02.213+00:00 2026-10-15T04:55:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:03.216+00:00 2026-10-15T04:55:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:04.221+00:00 2026-10-15T04:55:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:05.239+00:00 2026-10-15T04:55:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:06.232+00:00 2026-10-15T04:55:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:07.237+00:00 2026-10-15T04:55:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:08.243+00:00 2026-10-15T04:55:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:09.248+00:00 2026-10-15T04:55:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:10.264+00:00 2026-10-15T04:55:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:11.259+00:00 2026-10-15T04:55:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:12.263+00:00 2026-10-15T04:55:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:13.268+00:00 2026-10-15T04:55:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:14.321+00:00 2026-10-15T04:55:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:15.279+00:00 2026-10-15T04:55:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:16.284+00:00 2026-10-15T04:55:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:17.289+00:00 2026-10-15T04:55:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:18.294+00:00 2026-10-15T04:55:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:19.299+00:00 2026-10-15T04:55:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:20.304+00:00 2026-10-15T04:55:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:21.309+00:00 2026-10-15T04:55:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:22.322+00:00 2026-10-15T04:55:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:23.324+00:00 2026-10-15T04:55:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:24.323+00:00 2026-10-15T04:55:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:25.328+00:00 2026-10-15T04:55:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:26.334+00:00 2026-10-15T04:55:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:27.338+00:00 2026-10-15T04:55:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:28.403+00:00 2026-10-15T04:55:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:29.351+00:00 2026-10-15T04:55:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:30.355+00:00 2026-10-15T04:55:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:31.358+00:00 2026-10-15T04:55:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:32.363+00:00 2026-10-15T04:55:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:33.376+00:00 2026-10-15T04:55:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:34.378+00:00 2026-10-15T04:55:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:35.378+00:00 2026-10-15T04:55:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:36.383+00:00 2026-10-15T04:55:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:37.389+00:00 2026-10-15T04:55:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:38.397+00:00 2026-10-15T04:55:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:39.413+00:00 2026-10-15T04:55:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:40.403+00:00 2026-10-15T04:55:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:41.408+00:00 2026-10-15T04:55:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:42.416+00:00 2026-10-15T04:55:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:43.418+00:00 2026-10-15T04:55:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:44.424+00:00 2026-10-15T04:55:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:45.428+00:00 2026-10-15T04:55:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:46.433+00:00 2026-10-15T04:55:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:47.442+00:00 2026-10-15T04:55:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:48.443+00:00 2026-10-15T04:55:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:49.458+00:00 2026-10-15T04:55:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:50.453+00:00 2026-10-15T04:55:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:51.459+00:00 2026-10-15T04:55:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:52.463+00:00 2026-10-15T04:55:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:53.472+00:00 2026-10-15T04:55:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:54.474+00:00 2026-10-15T04:55:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:55.479+00:00 2026-10-15T04:55:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:56.483+00:00 2026-10-15T04:55:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:57.488+00:00 2026-10-15T04:55:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:58.492+00:00 2026-10-15T04:55:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:55:59.499+00:00 2026-10-15T04:55:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:00.503+00:00 2026-10-15T04:56:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:01.508+00:00 2026-10-15T04:56:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:02.514+00:00 2026-10-15T04:56:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:03.518+00:00 2026-10-15T04:56:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:04.524+00:00 2026-10-15T04:56:04Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:04.524+00:00 2026-10-15T04:56:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:05.528+00:00 2026-10-15T04:56:05Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:05.528+00:00 2026-10-15T04:56:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:06.549+00:00 2026-10-15T04:56:06Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:06.549+00:00 2026-10-15T04:56:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:07.539+00:00 2026-10-15T04:56:07Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:07.539+00:00 2026-10-15T04:56:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:08.558+00:00 2026-10-15T04:56:08Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:08.558+00:00 2026-10-15T04:56:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:09.549+00:00 2026-10-15T04:56:09Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:09.550+00:00 2026-10-15T04:56:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:10.555+00:00 2026-10-15T04:56:10Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:56:10.555+00:00 2026-10-15T04:56:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log new file mode 100644 index 00000000..78eb0719 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T04:47:52.102+00:00 2026-10-15T04:47:52.104023958Z {"level":"info","ts":"2026-10-15T04:47:52Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"8483800e-5d84-4a39-9698-b4f98903534e","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} +2026-10-15T04:51:15.356+00:00 2026-10-15T04:51:15.366719534Z {"level":"info","ts":"2026-10-15T04:51:15Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"a33a6ae4-616b-4e2d-b00c-0100362558de","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json new file mode 100644 index 00000000..fbc28464 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json @@ -0,0 +1,275 @@ +{ + "case": "after-A1-0.5b-l4-cold", + "collected": "2026-10-15T04:56:11.555+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 04:47:51.110000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen2.5-0.5B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 04:47:51.111000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 04:51:08.186000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "c34fd223-1dd8-443e-850e-68d16b0d897e" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 04:51:10.937000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 04:51:14.170000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 04:51:15.072000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 04:51:15.072000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 04:51:15.661000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 04:51:15.662000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 04:56:04.835000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4" + }, + "client.end": { + "ts": "2026-10-15 04:56:04.838000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 04:51:15+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 04:51:15.069000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 04:51:15.069000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 04:51:15.069000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 04:54:22+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 04:51:15+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 04:51:15+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 04:54:22+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 04:54:22+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 04:51:15+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 04:54:22+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 04:51:15+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "sampler.event_Pulling": { + "ts": "2026-10-15 04:51:19+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 04:54:21+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s inc", + "kubelet_pull_s": 182.5, + "pulled_from_cache": false + }, + "sampler.event_Created": { + "ts": "2026-10-15 04:54:21+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 04:54:22+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 04:54:45.242588+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 04:55:00.403987+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:00 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 04:55:02.093177+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:02 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 04:55:03.439031+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:03 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 04:55:04.519608+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:04 [default_loader.py:430] Loading weights took 1.08 seconds", + "self_reported_s": 1.08 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 04:55:05.323053+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:05 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.543823 seconds", + "self_reported_s": 3.543823 + }, + "sampler.compile_done": { + "ts": "2026-10-15 04:55:36.056714+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:36 [monitor.py:53] torch.compile took 28.53 s in total", + "self_reported_s": 28.53 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 04:55:51.460006+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:51 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", + "self_reported_s": 11.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 04:55:52.012063+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:52 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 04:56:02.777706+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:56:02 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.45 s", + "self_reported_s": 57.45 + }, + "sampler.ready_key": { + "ts": "2026-10-15 04:56:04+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + }, + "trainer": { + "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 04:47:51+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 04:47:51.664000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 04:47:51.664000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 04:47:53.304000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 04:50:45+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 04:47:52+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 04:47:52+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 04:50:45+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 04:50:45+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 04:47:52+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 04:50:45+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 04:47:52+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "trainer.event_Pulling": { + "ts": "2026-10-15 04:47:56+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 04:50:44+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.27", + "kubelet_pull_s": 168.27100000000002, + "pulled_from_cache": false + }, + "trainer.event_Created": { + "ts": "2026-10-15 04:50:44+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 04:50:45+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 04:50:52.077847+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 04:50:52.221389+00:00", + "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 04:51:07.261679+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 04:51:07.264149+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 04:51:08+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.md b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.md new file mode 100644 index 00000000..b0af26e8 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.md @@ -0,0 +1,111 @@ +# Cold start timeline: after-A1-0.5b-l4-cold + +t=0 is the client's `create_model` call at 2026-10-15T04:47:51.111+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen2-5-0-5b-0-trainer` on `gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.workload_created | 04:47:51.000 | -0.1 | | Workload creationTimestamp (apiserver) | +| trainer.workload_first_seen_by_poller | 04:47:51.664 | 0.6 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 04:47:51.664 | 0.6 | | workloads.tsv poller: status.podName set | +| trainer.pod_created | 04:47:52.000 | 0.9 | | pod creationTimestamp (apiserver) | +| trainer.pod_Initialized | 04:47:52.000 | 0.9 | | pod condition Initialized | +| trainer.pod_PodScheduled | 04:47:52.000 | 0.9 | | pod condition PodScheduled | +| trainer.event_Scheduled | 04:47:52.000 | 0.9 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 | +| trainer.workload_reports_node | 04:47:53.304 | 2.2 | | workloads.tsv poller: status.nodeName set | +| trainer.event_Pulling | 04:47:56.000 | 4.9 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" | +| trainer.event_Pulled | 04:50:44.000 | 172.9 | 168.27100000000002 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" in 2m48.271s (2m48.27 | +| trainer.event_Created | 04:50:44.000 | 172.9 | | event: Container created | +| trainer.condition_Placed_True | 04:50:45.000 | 173.9 | | Workload condition Placed | +| trainer.pod_Ready | 04:50:45.000 | 173.9 | | pod condition Ready | +| trainer.pod_ContainersReady | 04:50:45.000 | 173.9 | | pod condition ContainersReady | +| trainer.container_started | 04:50:45.000 | 173.9 | | containerStatuses.state.running.startedAt | +| trainer.event_Started | 04:50:45.000 | 173.9 | | event: Container started | +| trainer.process_first_log | 04:50:52.077 | 181.0 | | container log, first line | +| trainer.load_start | 04:50:52.221 | 181.1 | | container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0... | +| trainer.load_done | 04:51:07.261 | 196.2 | | container log: Successfully loaded. | +| trainer.processor_started | 04:51:07.264 | 196.2 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1). | +| trainer.ready_key | 04:51:08.000 | 196.9 | | redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B | + +## sampler: `orw-lora-qwen-qwen2-5-0-5b-0-sampler` on `gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_created | 04:51:15.000 | 203.9 | | Workload creationTimestamp (apiserver) | +| sampler.pod_created | 04:51:15.000 | 203.9 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 04:51:15.000 | 203.9 | | pod condition Initialized | +| sampler.pod_PodScheduled | 04:51:15.000 | 203.9 | | pod condition PodScheduled | +| sampler.event_Scheduled | 04:51:15.000 | 203.9 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 | +| sampler.workload_first_seen_by_poller | 04:51:15.069 | 204.0 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_reports_pod | 04:51:15.069 | 204.0 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 04:51:15.069 | 204.0 | | workloads.tsv poller: status.nodeName set | +| sampler.event_Pulling | 04:51:19.000 | 207.9 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" | +| sampler.event_Pulled | 04:54:21.000 | 389.9 | 182.5 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" in 3m2.5s (3m2.5s inc | +| sampler.event_Created | 04:54:21.000 | 389.9 | | event: Container created | +| sampler.condition_Placed_True | 04:54:22.000 | 390.9 | | Workload condition Placed | +| sampler.pod_Ready | 04:54:22.000 | 390.9 | | pod condition Ready | +| sampler.pod_ContainersReady | 04:54:22.000 | 390.9 | | pod condition ContainersReady | +| sampler.container_started | 04:54:22.000 | 390.9 | | containerStatuses.state.running.startedAt | +| sampler.event_Started | 04:54:22.000 | 390.9 | | event: Container started | +| sampler.process_first_log | 04:54:45.242 | 414.1 | | container log, first line | +| sampler.engine_init_start | 04:55:00.403 | 429.3 | | container log: (EngineCore pid=161) INFO 10-15 04:55:00 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 04:55:02.093 | 431.0 | | container log: (EngineCore pid=161) INFO 10-15 04:55:02 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 04:55:03.439 | 432.3 | | container log: (EngineCore pid=161) INFO 10-15 04:55:03 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 04:55:04.519 | 433.4 | 1.08 | container log: (EngineCore pid=161) INFO 10-15 04:55:04 [default_loader.py:430] Loading weights took 1.08 seconds | +| sampler.model_loaded | 04:55:05.323 | 434.2 | 3.543823 | container log: (EngineCore pid=161) INFO 10-15 04:55:05 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.543823 seconds | +| sampler.compile_done | 04:55:36.056 | 464.9 | 28.53 | container log: (EngineCore pid=161) INFO 10-15 04:55:36 [monitor.py:53] torch.compile took 28.53 s in total | +| sampler.graphs_captured | 04:55:51.460 | 480.3 | 11.0 | container log: (EngineCore pid=161) INFO 10-15 04:55:51 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB | +| sampler.kv_cache_sized | 04:55:52.012 | 480.9 | | container log: (EngineCore pid=161) INFO 10-15 04:55:52 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB | +| sampler.engine_init_done | 04:56:02.777 | 491.7 | 57.45 | container log: (EngineCore pid=161) INFO 10-15 04:56:02 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.45 s | +| sampler.ready_key | 04:56:04.000 | 492.9 | | redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 04:47:51.110 | -0.0 | +| client.create_model.start | 04:47:51.111 | 0.0 | +| client.create_model.done | 04:51:08.186 | 197.1 | +| client.forward_backward.start | 04:51:10.937 | 199.8 | +| client.forward_backward.done | 04:51:14.170 | 203.1 | +| client.optim_step.done | 04:51:15.072 | 204.0 | +| client.save_weights_and_get_sampling_client.start | 04:51:15.072 | 204.0 | +| client.save_weights_and_get_sampling_client.done | 04:51:15.661 | 204.6 | +| client.sample.start | 04:51:15.662 | 204.6 | +| client.sample.done | 04:56:04.835 | 493.7 | +| client.end | 04:56:04.838 | 493.7 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.9 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 172.0 | +| container create | trainer.event_Pulled | trainer.container_started | 1.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.2 | +| weights load | trainer.load_start | trainer.load_done | 15.0 | +| ready key | trainer.load_done | trainer.ready_key | 0.7 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 197.1 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.1 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 186.0 | +| container create | sampler.event_Pulled | sampler.container_started | 1.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 23.2 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 15.2 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.7 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.4 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.5 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.7 | +| ready key | sampler.engine_init_done | sampler.ready_key | 1.2 | +| client: first sample blocked | client.sample.start | client.sample.done | 289.2 | + diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt new file mode 100644 index 00000000..e34900e5 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T04:47:46.873+00:00 diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt new file mode 100644 index 00000000..a9917932 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json new file mode 100644 index 00000000..c15af16a --- /dev/null +++ b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040062551935001", + "uid": "0752a694-3e75-44aa-9348-ae1c3f3fff13" + }, + "spec": { + "accelerator": { + "memory": "4Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "ed5c4cfd-e94a-4a7f-a2a6-62ba293d8d04", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T04:54:22Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "4Gi", + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:51Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039845830863015", + "uid": "4bef5a35-3bf8-4444-9043-47f86b502e3e" + }, + "spec": { + "accelerator": { + "memory": "5Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "0772feaa-10d8-4243-ac6a-4b4ee0615042", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T04:50:45Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "5Gi", + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log new file mode 100644 index 00000000..8bf3bc56 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log @@ -0,0 +1,35 @@ +watching; recorders [14913, 14914, 14916, 14918, 14920, 14915, 14917, 14919] write into runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached +collected 2 worker log(s) into runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.2 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 1.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 3.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.8 | +| weights load | trainer.load_start | trainer.load_done | 2.5 | +| ready key | trainer.load_done | trainer.ready_key | -18.3 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 15.5 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.0 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 29.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 3.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 11.6 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 27.1 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.7 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.5 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.4 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.9 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.9 | +| client: first sample blocked | client.sample.start | client.sample.done | 134.6 | + diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log new file mode 100644 index 00000000..837468ea --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log @@ -0,0 +1,33 @@ +2026-10-15T04:57:20.742+00:00 2026-10-15T04:57:20.747394853Z INFO: 10.32.3.1:48902 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:57:22.081+00:00 2026-10-15T04:57:22.098120271Z INFO: 127.0.0.1:51376 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T04:57:22.332+00:00 2026-10-15T04:57:22.350492601Z INFO: 127.0.0.1:51380 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T04:57:22.399+00:00 2026-10-15T04:57:22.417496524Z INFO: 127.0.0.1:51380 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T04:57:22.642+00:00 2026-10-15T04:57:22.659983411Z [API_SERVER] enqueued op=create_model request_id=3c646264-eb51-4f2b-af28-b651e7b8d65f model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:57:22.642+00:00 2026-10-15T04:57:22.660591970Z INFO: 127.0.0.1:51390 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T04:57:30.732+00:00 2026-10-15T04:57:30.746832525Z INFO: 10.32.3.1:60502 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:57:32.582+00:00 2026-10-15T04:57:32.599215799Z INFO: 127.0.0.1:41916 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:57:37.537+00:00 2026-10-15T04:57:37.550418481Z INFO: 127.0.0.1:41936 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T04:57:40.140+00:00 2026-10-15T04:57:40.156908336Z [API_SERVER] enqueued op=forward_backward request_id=e5d06b8f-011b-4cc7-b00f-75a18d4cc68c model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:57:40.140+00:00 2026-10-15T04:57:40.157358753Z INFO: 127.0.0.1:41936 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T04:57:40.728+00:00 2026-10-15T04:57:40.746424135Z INFO: 10.32.3.1:52382 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:57:43.347+00:00 2026-10-15T04:57:43.364027632Z [API_SERVER] enqueued op=optim_step request_id=9c7bfa9d-8c2e-4163-8ddb-2d3872d13f25 model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:57:43.347+00:00 2026-10-15T04:57:43.364731477Z INFO: 127.0.0.1:41936 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T04:57:44.137+00:00 2026-10-15T04:57:44.154771060Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=32c6fd95-8437-4062-86d2-5fd031af54e4 model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T04:57:44.138+00:00 2026-10-15T04:57:44.156034597Z INFO: 127.0.0.1:41936 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T04:57:44.807+00:00 2026-10-15T04:57:44.824246114Z INFO: 127.0.0.1:35608 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T04:57:50.742+00:00 2026-10-15T04:57:50.746409736Z INFO: 10.32.3.1:46740 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:58:00.732+00:00 2026-10-15T04:58:00.747039608Z INFO: 10.32.3.1:49148 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:58:10.732+00:00 2026-10-15T04:58:10.747575732Z INFO: 10.32.3.1:53698 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:58:20.731+00:00 2026-10-15T04:58:20.747432364Z INFO: 10.32.3.1:58312 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:58:30.731+00:00 2026-10-15T04:58:30.747504088Z INFO: 10.32.3.1:52920 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:58:32.787+00:00 2026-10-15T04:58:32.804434225Z INFO: 127.0.0.1:46778 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:58:40.730+00:00 2026-10-15T04:58:40.746974548Z INFO: 10.32.3.1:46976 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:58:50.730+00:00 2026-10-15T04:58:50.747141544Z INFO: 10.32.3.1:59494 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:59:00.731+00:00 2026-10-15T04:59:00.747576147Z INFO: 10.32.3.1:43150 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:59:10.730+00:00 2026-10-15T04:59:10.747345321Z INFO: 10.32.3.1:32882 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:59:20.732+00:00 2026-10-15T04:59:20.747511163Z INFO: 10.32.3.1:54978 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:59:22.982+00:00 2026-10-15T04:59:23.000147720Z INFO: 127.0.0.1:39214 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T04:59:30.737+00:00 2026-10-15T04:59:30.746624775Z INFO: 10.32.3.1:33562 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:59:40.731+00:00 2026-10-15T04:59:40.747472732Z INFO: 10.32.3.1:56128 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T04:59:50.729+00:00 2026-10-15T04:59:50.746593670Z INFO: 10.32.3.1:54218 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:00:00.731+00:00 2026-10-15T05:00:00.747026021Z INFO: 10.32.3.1:37274 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl new file mode 100644 index 00000000..6f50fdd1 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T04:57:21.804+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T04:57:21.804+00:00", "t": 0.001, "phase": "create_model.start"} +{"ts": "2026-10-15T04:57:37.299+00:00", "t": 15.495, "phase": "create_model.done", "model_id": "3c646264-eb51-4f2b-af28-b651e7b8d65f"} +{"ts": "2026-10-15T04:57:40.066+00:00", "t": 18.262, "phase": "forward_backward.start"} +{"ts": "2026-10-15T04:57:43.279+00:00", "t": 21.475, "phase": "forward_backward.done"} +{"ts": "2026-10-15T04:57:44.032+00:00", "t": 22.228, "phase": "optim_step.done"} +{"ts": "2026-10-15T04:57:44.033+00:00", "t": 22.229, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T04:57:44.610+00:00", "t": 22.807, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T04:57:44.612+00:00", "t": 22.808, "phase": "sample.start"} +{"ts": "2026-10-15T04:59:59.246+00:00", "t": 157.443, "phase": "sample.done", "text": " 4"} +{"ts": "2026-10-15T04:59:59.247+00:00", "t": 157.443, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log new file mode 100644 index 00000000..8738ab7c --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.001s create_model.start +[cold-start] 15.495s create_model.done {'model_id': '3c646264-eb51-4f2b-af28-b651e7b8d65f'} +[cold-start] 18.262s forward_backward.start +[cold-start] 21.475s forward_backward.done +[cold-start] 22.228s optim_step.done +[cold-start] 22.229s save_weights_and_get_sampling_client.start +[cold-start] 22.807s save_weights_and_get_sampling_client.done +[cold-start] 22.808s sample.start +[cold-start] 157.443s sample.done {'text': ' 4'} +[cold-start] 157.443s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt new file mode 100644 index 00000000..2cb3aa22 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T04:59:59.733+00:00 diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json new file mode 100644 index 00000000..f67d42a0 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json @@ -0,0 +1,1699 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:10Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:10Z", + "name": "cold-start-fio.18de9a0ef295773d", + "namespace": "openrl-system", + "resourceVersion": "1792039090760383011", + "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5be63862", + "namespace": "openrl-system", + "resourceVersion": "1792039092515007000", + "uid": "355b59b9-3661-499d-8640-3f40c3c89375" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5ecfa51a", + "namespace": "openrl-system", + "resourceVersion": "1792039092561663021", + "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f61ddd1e4", + "namespace": "openrl-system", + "resourceVersion": "1792039092605935011", + "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:40:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:40:41Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:40:41Z", + "name": "cold-start-fio.18de9a32081cf908", + "namespace": "openrl-system", + "resourceVersion": "1792039241446287009", + "uid": "14a62e42-13ba-4526-a04d-a934702e1163" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:18Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:18Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:18Z", + "name": "cold-start-fio.18de9a3a9867ce9f", + "namespace": "openrl-system", + "resourceVersion": "1792039278208207001", + "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad2bcac91", + "namespace": "openrl-system", + "resourceVersion": "1792039279188751014", + "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad378cd74", + "namespace": "openrl-system", + "resourceVersion": "1792039279219679006", + "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad5aed221", + "namespace": "openrl-system", + "resourceVersion": "1792039279235135013", + "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:11Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:11Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:11Z", + "name": "cold-start-fio.18de9a70cd2c6a68", + "namespace": "openrl-system", + "resourceVersion": "1792039511048479001", + "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "resourceVersion": "1791941114537823021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Stopping container api-server", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", + "namespace": "openrl-system", + "resourceVersion": "1792039545418735013", + "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "namespace": "openrl-system", + "resourceVersion": "1792039545300351007", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", + "namespace": "openrl-system", + "resourceVersion": "1792039545387567001", + "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" + }, + "reason": "SuccessfulDelete", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546048591003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", + "namespace": "openrl-system", + "resourceVersion": "1792039546102351012", + "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:49Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:49Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", + "namespace": "openrl-system", + "resourceVersion": "1792039549712335019", + "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", + "namespace": "openrl-system", + "resourceVersion": "1792039558721215017", + "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", + "namespace": "openrl-system", + "resourceVersion": "1792039558783199013", + "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", + "namespace": "openrl-system", + "resourceVersion": "1792039558862303009", + "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "namespace": "openrl-system", + "resourceVersion": "1792039545940671018", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", + "namespace": "openrl-system", + "resourceVersion": "1792039546068975009", + "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" + }, + "reason": "SuccessfulCreate", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545232655018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78c9668cf7", + "namespace": "openrl-system", + "resourceVersion": "1792039545327119005", + "uid": "7b1433df-3524-478b-aba2-8979950e245e" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545464319018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78ef8b9e1b", + "namespace": "openrl-system", + "resourceVersion": "1792039545956079020", + "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:15Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875490623001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:15Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", + "namespace": "openrl-system", + "resourceVersion": "1792039875609279013", + "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:19Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:19Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", + "namespace": "openrl-system", + "resourceVersion": "1792039879126319008", + "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", + "namespace": "openrl-system", + "resourceVersion": "1792040061645247020", + "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", + "namespace": "openrl-system", + "resourceVersion": "1792040061849391002", + "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", + "namespace": "openrl-system", + "resourceVersion": "1792040062252079023", + "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", + "namespace": "openrl-system", + "resourceVersion": "1792040192780783021", + "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264403887001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", + "namespace": "openrl-system", + "resourceVersion": "1792040264444463021", + "uid": "a85006a2-68a6-4919-b148-08d58b546d07" + }, + "reason": "FailedScheduling", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Warning" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264434463001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", + "namespace": "openrl-system", + "resourceVersion": "1792040264884223002", + "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" + }, + "reason": "NotTriggerScaleUp", + "reportingComponent": "cluster-autoscaler", + "reportingInstance": "", + "source": { + "component": "cluster-autoscaler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264893039001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", + "namespace": "openrl-system", + "resourceVersion": "1792040293171567001", + "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", + "namespace": "openrl-system", + "resourceVersion": "1792040296672127014", + "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", + "namespace": "openrl-system", + "resourceVersion": "1792040296703007017", + "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", + "namespace": "openrl-system", + "resourceVersion": "1792040296889487014", + "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:52Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672220015012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:52Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", + "namespace": "openrl-system", + "resourceVersion": "1792039672350591019", + "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:56Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:56Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:56Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", + "namespace": "openrl-system", + "resourceVersion": "1792039676166239015", + "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", + "namespace": "openrl-system", + "resourceVersion": "1792039844457167000", + "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", + "namespace": "openrl-system", + "resourceVersion": "1792039844632591021", + "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:45Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:45Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", + "namespace": "openrl-system", + "resourceVersion": "1792039845170959016", + "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", + "namespace": "openrl-system", + "resourceVersion": "1792040192981183013", + "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:23Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040242926223012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:23Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:23Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", + "namespace": "openrl-system", + "resourceVersion": "1792040243043743024", + "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", + "namespace": "openrl-system", + "resourceVersion": "1792040246545487023", + "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", + "namespace": "openrl-system", + "resourceVersion": "1792040246576895006", + "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", + "namespace": "openrl-system", + "resourceVersion": "1792040246742815002", + "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", + "namespace": "openrl-system", + "resourceVersion": "1792038835732623016", + "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", + "namespace": "openrl-system", + "resourceVersion": "1792038835415695012", + "uid": "918abc9a-1f24-4767-8928-b202482f11ed" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt new file mode 100644 index 00000000..0b795ec8 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt @@ -0,0 +1,119 @@ +2026-10-15T04:58:27.584176783Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T04:58:39.659845746Z INFO 10-15 04:58:39 [model.py:692] Resolved architecture: Qwen2ForCausalLM +2026-10-15T04:58:39.660367167Z INFO 10-15 04:58:39 [model.py:2030] Using max model len 8192 +2026-10-15T04:58:39.662546840Z INFO 10-15 04:58:39 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. +2026-10-15T04:58:40.049753461Z INFO 10-15 04:58:40 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T04:58:43.331466819Z WARNING 10-15 04:58:43 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T04:58:54.733167125Z (EngineCore pid=161) INFO 10-15 04:58:54 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T04:58:55.532460244Z (EngineCore pid=161) INFO 10-15 04:58:55 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_8fc70d1b7a3d4bf98e7322fa6a9dbb46 backend=nccl +2026-10-15T04:58:55.539943546Z (EngineCore pid=161) INFO 10-15 04:58:55 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T04:58:55.539970703Z (EngineCore pid=161) INFO 10-15 04:58:55 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T04:58:56.402021183Z (EngineCore pid=161) INFO 10-15 04:58:56 [model_runner.py:396] Loading model from scratch... +2026-10-15T04:58:57.269734349Z (EngineCore pid=161) INFO 10-15 04:58:57 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T04:58:57.270211979Z (EngineCore pid=161) INFO 10-15 04:58:57 [flash_attn.py:1116] Using FlashAttention version 2 +2026-10-15T04:58:57.386036165Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T04:58:57.386373759Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T04:58:57.753318801Z (EngineCore pid=161) INFO 10-15 04:58:57 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.62 GiB. +2026-10-15T04:58:57.753358302Z (EngineCore pid=161) INFO 10-15 04:58:57 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T04:58:57.754786362Z (EngineCore pid=161) +Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T04:57:33.682578439Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T04:57:33.682611900Z +2026-10-15T04:57:33.802165728Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... +2026-10-15T04:57:33.918116197Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T04:57:34.743979630Z [prefetch] reading 1 file(s), 0.92 GiB, from nfs into the page cache with 8 threads +2026-10-15T04:57:35.446752761Z [prefetch] done: 0.92 GiB in 0.7 s (1341 MiB/s) +2026-10-15T04:57:36.264734680Z +Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json new file mode 100644 index 00000000..6a4ea21e --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json @@ -0,0 +1,6786 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "generateName": "open-rl-api-server-6d86b498f5-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "6d86b498f5", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + } + ], + "resourceVersion": "1792039570790271003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2ns79", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:59Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:45:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.11", + "podIPs": [ + { + "ip": "10.32.3.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:45:46Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "cloud.google.com/cluster_autoscaler_unhelpable_since": "2026-10-15T04:57:44+0000", + "cloud.google.com/cluster_autoscaler_unhelpable_until": "Inf" + }, + "creationTimestamp": "2026-10-15T04:57:44Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "uid": "4ce140e7-4f1e-4b14-9877-bb659f6e0343" + } + ], + "resourceVersion": "1792040297125103001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "2ee5c8bd-e346-48b6-8cb1-eb62a873538d" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j4zlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-j4zlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:58:17Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:58:13Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:58:17Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:58:17Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:58:13Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "25Gi" + }, + "containerID": "containerd://9da0479c74bfc8042a71c292922826c6ce3ad90ac3ac5cb3b018c1f4cff943a0", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:58:16Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j4zlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.0.21", + "podIPs": [ + { + "ip": "10.32.0.21" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:58:13Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:22Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "uid": "35aa092e-27db-4eca-ad32-e9e0d6070c77" + } + ], + "resourceVersion": "1792040246938655012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "efac33f1-a138-4d61-b066-ff1ef1279fec" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-gvbfv", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-gvbfv", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:57:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:57:23Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:57:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:57:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:57:23Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "21Gi" + }, + "containerID": "containerd://2d4343c24b9a9775657cfe3e8106b8ddd769515df55d3f40bbcc45bca83bbd69", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:57:26Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-gvbfv", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.13.27", + "podIPs": [ + { + "ip": "10.32.13.27" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:57:23Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log new file mode 100644 index 00000000..d60c3560 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log @@ -0,0 +1,175 @@ +2026-10-15T04:57:18.742+00:00 2026-10-15T04:57:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:19.743+00:00 2026-10-15T04:57:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:20.748+00:00 2026-10-15T04:57:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:21.753+00:00 2026-10-15T04:57:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:22.758+00:00 2026-10-15T04:57:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:23.763+00:00 2026-10-15T04:57:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:24.768+00:00 2026-10-15T04:57:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:25.774+00:00 2026-10-15T04:57:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:26.778+00:00 2026-10-15T04:57:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:27.784+00:00 2026-10-15T04:57:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:28.790+00:00 2026-10-15T04:57:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:29.796+00:00 2026-10-15T04:57:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:30.800+00:00 2026-10-15T04:57:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:31.806+00:00 2026-10-15T04:57:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:32.811+00:00 2026-10-15T04:57:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:33.815+00:00 2026-10-15T04:57:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:34.820+00:00 2026-10-15T04:57:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:35.826+00:00 2026-10-15T04:57:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:36.830+00:00 2026-10-15T04:57:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:37.835+00:00 2026-10-15T04:57:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:38.840+00:00 2026-10-15T04:57:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:39.846+00:00 2026-10-15T04:57:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:40.851+00:00 2026-10-15T04:57:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:41.857+00:00 2026-10-15T04:57:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:42.862+00:00 2026-10-15T04:57:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:43.866+00:00 2026-10-15T04:57:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:44.870+00:00 2026-10-15T04:57:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:45.875+00:00 2026-10-15T04:57:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:46.879+00:00 2026-10-15T04:57:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:47.906+00:00 2026-10-15T04:57:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:48.893+00:00 2026-10-15T04:57:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:49.897+00:00 2026-10-15T04:57:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:50.901+00:00 2026-10-15T04:57:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:51.907+00:00 2026-10-15T04:57:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:52.911+00:00 2026-10-15T04:57:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:53.917+00:00 2026-10-15T04:57:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:54.921+00:00 2026-10-15T04:57:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:55.928+00:00 2026-10-15T04:57:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:56.946+00:00 2026-10-15T04:57:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:57.941+00:00 2026-10-15T04:57:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:58.944+00:00 2026-10-15T04:57:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:57:59.953+00:00 2026-10-15T04:57:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:00.953+00:00 2026-10-15T04:58:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:01.958+00:00 2026-10-15T04:58:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:02.972+00:00 2026-10-15T04:58:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:03.974+00:00 2026-10-15T04:58:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:04.981+00:00 2026-10-15T04:58:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:05.980+00:00 2026-10-15T04:58:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:06.985+00:00 2026-10-15T04:58:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:07.991+00:00 2026-10-15T04:58:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:08.996+00:00 2026-10-15T04:58:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:10.027+00:00 2026-10-15T04:58:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:11.015+00:00 2026-10-15T04:58:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:12.011+00:00 2026-10-15T04:58:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:13.099+00:00 2026-10-15T04:58:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:14.021+00:00 2026-10-15T04:58:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:15.026+00:00 2026-10-15T04:58:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:16.031+00:00 2026-10-15T04:58:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:17.206+00:00 2026-10-15T04:58:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:18.041+00:00 2026-10-15T04:58:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:19.059+00:00 2026-10-15T04:58:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:20.052+00:00 2026-10-15T04:58:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:21.062+00:00 2026-10-15T04:58:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:22.064+00:00 2026-10-15T04:58:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:23.067+00:00 2026-10-15T04:58:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:24.077+00:00 2026-10-15T04:58:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:25.078+00:00 2026-10-15T04:58:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:26.110+00:00 2026-10-15T04:58:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:27.093+00:00 2026-10-15T04:58:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:28.097+00:00 2026-10-15T04:58:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:29.108+00:00 2026-10-15T04:58:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:30.111+00:00 2026-10-15T04:58:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:31.112+00:00 2026-10-15T04:58:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:32.117+00:00 2026-10-15T04:58:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:33.122+00:00 2026-10-15T04:58:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:34.127+00:00 2026-10-15T04:58:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:35.171+00:00 2026-10-15T04:58:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:36.137+00:00 2026-10-15T04:58:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:37.143+00:00 2026-10-15T04:58:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:38.148+00:00 2026-10-15T04:58:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:39.153+00:00 2026-10-15T04:58:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:40.161+00:00 2026-10-15T04:58:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:41.163+00:00 2026-10-15T04:58:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:42.168+00:00 2026-10-15T04:58:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:43.173+00:00 2026-10-15T04:58:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:44.179+00:00 2026-10-15T04:58:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:45.183+00:00 2026-10-15T04:58:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:46.290+00:00 2026-10-15T04:58:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:47.193+00:00 2026-10-15T04:58:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:48.198+00:00 2026-10-15T04:58:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:49.204+00:00 2026-10-15T04:58:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:50.208+00:00 2026-10-15T04:58:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:51.394+00:00 2026-10-15T04:58:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:52.218+00:00 2026-10-15T04:58:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:53.225+00:00 2026-10-15T04:58:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:54.241+00:00 2026-10-15T04:58:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:55.234+00:00 2026-10-15T04:58:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:56.239+00:00 2026-10-15T04:58:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:57.245+00:00 2026-10-15T04:58:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:58.257+00:00 2026-10-15T04:58:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:58:59.256+00:00 2026-10-15T04:58:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:00.269+00:00 2026-10-15T04:59:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:01.271+00:00 2026-10-15T04:59:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:02.271+00:00 2026-10-15T04:59:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:03.287+00:00 2026-10-15T04:59:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:04.283+00:00 2026-10-15T04:59:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:05.286+00:00 2026-10-15T04:59:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:06.291+00:00 2026-10-15T04:59:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:07.296+00:00 2026-10-15T04:59:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:08.300+00:00 2026-10-15T04:59:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:09.310+00:00 2026-10-15T04:59:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:10.311+00:00 2026-10-15T04:59:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:11.316+00:00 2026-10-15T04:59:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:12.322+00:00 2026-10-15T04:59:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:13.327+00:00 2026-10-15T04:59:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:14.331+00:00 2026-10-15T04:59:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:15.337+00:00 2026-10-15T04:59:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:16.344+00:00 2026-10-15T04:59:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:17.348+00:00 2026-10-15T04:59:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:18.352+00:00 2026-10-15T04:59:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:19.357+00:00 2026-10-15T04:59:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:20.366+00:00 2026-10-15T04:59:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:21.368+00:00 2026-10-15T04:59:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:22.377+00:00 2026-10-15T04:59:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:23.379+00:00 2026-10-15T04:59:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:24.384+00:00 2026-10-15T04:59:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:25.390+00:00 2026-10-15T04:59:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:26.395+00:00 2026-10-15T04:59:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:27.403+00:00 2026-10-15T04:59:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:28.404+00:00 2026-10-15T04:59:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:29.410+00:00 2026-10-15T04:59:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:30.415+00:00 2026-10-15T04:59:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:31.420+00:00 2026-10-15T04:59:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:32.426+00:00 2026-10-15T04:59:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:33.432+00:00 2026-10-15T04:59:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:34.437+00:00 2026-10-15T04:59:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:35.445+00:00 2026-10-15T04:59:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:36.449+00:00 2026-10-15T04:59:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:37.453+00:00 2026-10-15T04:59:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:38.458+00:00 2026-10-15T04:59:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:39.463+00:00 2026-10-15T04:59:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:40.469+00:00 2026-10-15T04:59:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:41.474+00:00 2026-10-15T04:59:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:42.478+00:00 2026-10-15T04:59:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:43.484+00:00 2026-10-15T04:59:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:44.490+00:00 2026-10-15T04:59:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:45.494+00:00 2026-10-15T04:59:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:46.500+00:00 2026-10-15T04:59:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:47.508+00:00 2026-10-15T04:59:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:48.509+00:00 2026-10-15T04:59:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:49.514+00:00 2026-10-15T04:59:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:50.519+00:00 2026-10-15T04:59:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:51.532+00:00 2026-10-15T04:59:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:52.529+00:00 2026-10-15T04:59:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:53.535+00:00 2026-10-15T04:59:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:54.545+00:00 2026-10-15T04:59:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:55.544+00:00 2026-10-15T04:59:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:56.565+00:00 2026-10-15T04:59:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:57.556+00:00 2026-10-15T04:59:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:58.561+00:00 2026-10-15T04:59:58Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:58.562+00:00 2026-10-15T04:59:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:59.566+00:00 2026-10-15T04:59:59Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T04:59:59.566+00:00 2026-10-15T04:59:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:00.571+00:00 2026-10-15T05:00:00Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:00.571+00:00 2026-10-15T05:00:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:01.590+00:00 2026-10-15T05:00:01Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:01.590+00:00 2026-10-15T05:00:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:02.582+00:00 2026-10-15T05:00:02Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:02.582+00:00 2026-10-15T05:00:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:03.588+00:00 2026-10-15T05:00:03Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:03.588+00:00 2026-10-15T05:00:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:04.600+00:00 2026-10-15T05:00:04Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:04.600+00:00 2026-10-15T05:00:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:05.599+00:00 2026-10-15T05:00:05Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:00:05.599+00:00 2026-10-15T05:00:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log new file mode 100644 index 00000000..0a6b928c --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T04:57:22.772+00:00 2026-10-15T04:57:22.789786341Z {"level":"info","ts":"2026-10-15T04:57:22Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"a299badc-c2da-43ab-b006-1bbc6eef5dd0","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} +2026-10-15T04:57:44.281+00:00 2026-10-15T04:57:44.297614785Z {"level":"info","ts":"2026-10-15T04:57:44Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"ac3b61c9-8905-4726-9ca8-f88e8b72698f","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json new file mode 100644 index 00000000..d90e4c40 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json @@ -0,0 +1,275 @@ +{ + "case": "after-B1-0.5b-l4-cached", + "collected": "2026-10-15T05:00:05.835+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 04:57:21.804000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen2.5-0.5B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 04:57:21.804000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 04:57:37.299000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "3c646264-eb51-4f2b-af28-b651e7b8d65f" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 04:57:40.066000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 04:57:43.279000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 04:57:44.032000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 04:57:44.033000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 04:57:44.610000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 04:57:44.612000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 04:59:59.246000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4" + }, + "client.end": { + "ts": "2026-10-15 04:59:59.247000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 04:57:44+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 04:57:44.625000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 04:57:44.625000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 04:58:13.579000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 04:58:17+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 04:57:44+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 04:58:13+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 04:58:17+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 04:58:17+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 04:58:13+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 04:58:16+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_FailedScheduling": { + "ts": "2026-10-15 04:57:44+00:00", + "source": "event: 0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, p" + }, + "sampler.event_NotTriggerScaleUp": { + "ts": "2026-10-15 04:57:44+00:00", + "source": "event: Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 04:58:13+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 04:58:16+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "sampler.event_Created": { + "ts": "2026-10-15 04:58:16+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 04:58:16+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 04:58:27.584176+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 04:58:54.733167+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:54 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 04:58:56.402021+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:56 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 04:58:57.753358+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:57 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 04:58:58.859754+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:58 [default_loader.py:430] Loading weights took 1.11 seconds", + "self_reported_s": 1.11 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 04:58:59.653997+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:59 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.562917 seconds", + "self_reported_s": 3.562917 + }, + "sampler.compile_done": { + "ts": "2026-10-15 04:59:30.263531+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:30 [monitor.py:53] torch.compile took 28.39 s in total", + "self_reported_s": 28.39 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 04:59:45.735209+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:45 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", + "self_reported_s": 11.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 04:59:46.273450+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:46 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 04:59:57.142509+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:57 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.48 s", + "self_reported_s": 57.48 + }, + "sampler.ready_key": { + "ts": "2026-10-15 04:59:58+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + }, + "trainer": { + "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 04:57:22+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 04:57:22.498000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 04:57:22.498000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 04:57:22.498000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 04:57:27+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 04:57:22+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 04:57:23+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 04:57:26+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 04:57:26+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 04:57:23+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 04:57:26+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 04:57:23+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 04:57:26+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "trainer.event_Created": { + "ts": "2026-10-15 04:57:26+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 04:57:26+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 04:57:33.659925+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 04:57:33.802165+00:00", + "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 04:57:36.346688+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 04:57:36.349344+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 04:57:18+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.md b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.md new file mode 100644 index 00000000..eca7bddb --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.md @@ -0,0 +1,111 @@ +# Cold start timeline: after-B1-0.5b-l4-cached + +t=0 is the client's `create_model` call at 2026-10-15T04:57:21.804+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen2-5-0-5b-0-trainer` on `gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.ready_key | 04:57:18.000 | -3.8 | | redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B | +| trainer.workload_created | 04:57:22.000 | 0.2 | | Workload creationTimestamp (apiserver) | +| trainer.pod_created | 04:57:22.000 | 0.2 | | pod creationTimestamp (apiserver) | +| trainer.workload_first_seen_by_poller | 04:57:22.498 | 0.7 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 04:57:22.498 | 0.7 | | workloads.tsv poller: status.podName set | +| trainer.workload_reports_node | 04:57:22.498 | 0.7 | | workloads.tsv poller: status.nodeName set | +| trainer.pod_Initialized | 04:57:23.000 | 1.2 | | pod condition Initialized | +| trainer.pod_PodScheduled | 04:57:23.000 | 1.2 | | pod condition PodScheduled | +| trainer.event_Scheduled | 04:57:23.000 | 1.2 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 | +| trainer.pod_Ready | 04:57:26.000 | 4.2 | | pod condition Ready | +| trainer.pod_ContainersReady | 04:57:26.000 | 4.2 | | pod condition ContainersReady | +| trainer.container_started | 04:57:26.000 | 4.2 | | containerStatuses.state.running.startedAt | +| trainer.event_Pulled | 04:57:26.000 | 4.2 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| trainer.event_Created | 04:57:26.000 | 4.2 | | event: Container created | +| trainer.event_Started | 04:57:26.000 | 4.2 | | event: Container started | +| trainer.condition_Placed_True | 04:57:27.000 | 5.2 | | Workload condition Placed | +| trainer.process_first_log | 04:57:33.659 | 11.9 | | container log, first line | +| trainer.load_start | 04:57:33.802 | 12.0 | | container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0... | +| trainer.load_done | 04:57:36.346 | 14.5 | | container log: Successfully loaded. | +| trainer.processor_started | 04:57:36.349 | 14.5 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1). | + +## sampler: `orw-lora-qwen-qwen2-5-0-5b-0-sampler` on `gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_created | 04:57:44.000 | 22.2 | | Workload creationTimestamp (apiserver) | +| sampler.pod_created | 04:57:44.000 | 22.2 | | pod creationTimestamp (apiserver) | +| sampler.event_FailedScheduling | 04:57:44.000 | 22.2 | | event: 0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, p | +| sampler.event_NotTriggerScaleUp | 04:57:44.000 | 22.2 | | event: Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s) | +| sampler.workload_first_seen_by_poller | 04:57:44.625 | 22.8 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_reports_pod | 04:57:44.625 | 22.8 | | workloads.tsv poller: status.podName set | +| sampler.pod_Initialized | 04:58:13.000 | 51.2 | | pod condition Initialized | +| sampler.pod_PodScheduled | 04:58:13.000 | 51.2 | | pod condition PodScheduled | +| sampler.event_Scheduled | 04:58:13.000 | 51.2 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 | +| sampler.workload_reports_node | 04:58:13.579 | 51.8 | | workloads.tsv poller: status.nodeName set | +| sampler.container_started | 04:58:16.000 | 54.2 | | containerStatuses.state.running.startedAt | +| sampler.event_Pulled | 04:58:16.000 | 54.2 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| sampler.event_Created | 04:58:16.000 | 54.2 | | event: Container created | +| sampler.event_Started | 04:58:16.000 | 54.2 | | event: Container started | +| sampler.condition_Placed_True | 04:58:17.000 | 55.2 | | Workload condition Placed | +| sampler.pod_Ready | 04:58:17.000 | 55.2 | | pod condition Ready | +| sampler.pod_ContainersReady | 04:58:17.000 | 55.2 | | pod condition ContainersReady | +| sampler.process_first_log | 04:58:27.584 | 65.8 | | container log, first line | +| sampler.engine_init_start | 04:58:54.733 | 92.9 | | container log: (EngineCore pid=161) INFO 10-15 04:58:54 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 04:58:56.402 | 94.6 | | container log: (EngineCore pid=161) INFO 10-15 04:58:56 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 04:58:57.753 | 95.9 | | container log: (EngineCore pid=161) INFO 10-15 04:58:57 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 04:58:58.859 | 97.1 | 1.11 | container log: (EngineCore pid=161) INFO 10-15 04:58:58 [default_loader.py:430] Loading weights took 1.11 seconds | +| sampler.model_loaded | 04:58:59.653 | 97.8 | 3.562917 | container log: (EngineCore pid=161) INFO 10-15 04:58:59 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.562917 seconds | +| sampler.compile_done | 04:59:30.263 | 128.5 | 28.39 | container log: (EngineCore pid=161) INFO 10-15 04:59:30 [monitor.py:53] torch.compile took 28.39 s in total | +| sampler.graphs_captured | 04:59:45.735 | 143.9 | 11.0 | container log: (EngineCore pid=161) INFO 10-15 04:59:45 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB | +| sampler.kv_cache_sized | 04:59:46.273 | 144.5 | | container log: (EngineCore pid=161) INFO 10-15 04:59:46 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB | +| sampler.engine_init_done | 04:59:57.142 | 155.3 | 57.48 | container log: (EngineCore pid=161) INFO 10-15 04:59:57 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.48 s | +| sampler.ready_key | 04:59:58.000 | 156.2 | | redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 04:57:21.804 | 0.0 | +| client.create_model.start | 04:57:21.804 | 0.0 | +| client.create_model.done | 04:57:37.299 | 15.5 | +| client.forward_backward.start | 04:57:40.066 | 18.3 | +| client.forward_backward.done | 04:57:43.279 | 21.5 | +| client.optim_step.done | 04:57:44.032 | 22.2 | +| client.save_weights_and_get_sampling_client.start | 04:57:44.033 | 22.2 | +| client.save_weights_and_get_sampling_client.done | 04:57:44.610 | 22.8 | +| client.sample.start | 04:57:44.612 | 22.8 | +| client.sample.done | 04:59:59.246 | 157.4 | +| client.end | 04:59:59.247 | 157.4 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.2 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 1.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 3.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.8 | +| weights load | trainer.load_start | trainer.load_done | 2.5 | +| ready key | trainer.load_done | trainer.ready_key | -18.3 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 15.5 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.0 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 29.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 3.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 11.6 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 27.1 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.7 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.5 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.4 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.9 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.9 | +| client: first sample blocked | client.sample.start | client.sample.done | 134.6 | + diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt new file mode 100644 index 00000000..6982d578 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T04:57:17.557+00:00 diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt new file mode 100644 index 00000000..a9917932 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json new file mode 100644 index 00000000..8e52f6bf --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040297218207001", + "uid": "4ce140e7-4f1e-4b14-9877-bb659f6e0343" + }, + "spec": { + "accelerator": { + "memory": "4Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "2ee5c8bd-e346-48b6-8cb1-eb62a873538d", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T04:58:17Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "4Gi", + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:22Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040247036111015", + "uid": "35aa092e-27db-4eca-ad32-e9e0d6070c77" + }, + "spec": { + "accelerator": { + "memory": "5Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "efac33f1-a138-4d61-b066-ff1ef1279fec", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T04:57:27Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "5Gi", + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log new file mode 100644 index 00000000..7473ff1d --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log @@ -0,0 +1,35 @@ +watching; recorders [15841, 15842, 15844, 15846, 15848, 15843, 15845, 15847] write into runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached +collected 2 worker log(s) into runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.4 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 8.0 | +| weights load | trainer.load_start | trainer.load_done | 2.5 | +| ready key | trainer.load_done | trainer.ready_key | -18.5 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 15.8 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.2 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 23.3 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 16.3 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.8 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.6 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.3 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.8 | +| ready key | sampler.engine_init_done | sampler.ready_key | 1.9 | +| client: first sample blocked | client.sample.start | client.sample.done | 107.4 | + diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log new file mode 100644 index 00000000..df117526 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log @@ -0,0 +1,30 @@ +2026-10-15T05:01:15.885+00:00 2026-10-15T05:01:15.899417391Z INFO: 127.0.0.1:35950 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T05:01:16.112+00:00 2026-10-15T05:01:16.130561464Z INFO: 127.0.0.1:35952 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T05:01:16.180+00:00 2026-10-15T05:01:16.199505092Z INFO: 127.0.0.1:35952 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T05:01:16.443+00:00 2026-10-15T05:01:16.462704698Z [API_SERVER] enqueued op=create_model request_id=840431b4-6617-4074-8848-cf95526c9aec model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T05:01:16.445+00:00 2026-10-15T05:01:16.464180264Z INFO: 127.0.0.1:35962 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T05:01:20.728+00:00 2026-10-15T05:01:20.746712614Z INFO: 10.32.3.1:43472 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:01:26.372+00:00 2026-10-15T05:01:26.389349362Z INFO: 127.0.0.1:48808 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:01:30.732+00:00 2026-10-15T05:01:30.748033533Z INFO: 10.32.3.1:45936 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:01:31.605+00:00 2026-10-15T05:01:31.624783860Z INFO: 127.0.0.1:42044 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T05:01:34.338+00:00 2026-10-15T05:01:34.357162387Z [API_SERVER] enqueued op=forward_backward request_id=d985e84c-f973-4fc5-84a8-b078e63ccf69 model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T05:01:34.339+00:00 2026-10-15T05:01:34.357733525Z INFO: 127.0.0.1:42044 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T05:01:37.559+00:00 2026-10-15T05:01:37.576296735Z [API_SERVER] enqueued op=optim_step request_id=d3a96a54-0418-44fe-a5c3-2f1e6be71f7b model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T05:01:37.559+00:00 2026-10-15T05:01:37.576838072Z INFO: 127.0.0.1:42044 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T05:01:38.342+00:00 2026-10-15T05:01:38.358865914Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=963d0bbf-ae16-4566-b427-cfcb9f37e811 model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 +2026-10-15T05:01:38.342+00:00 2026-10-15T05:01:38.359362687Z INFO: 127.0.0.1:42044 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T05:01:39.001+00:00 2026-10-15T05:01:39.020317120Z INFO: 127.0.0.1:42052 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T05:01:40.729+00:00 2026-10-15T05:01:40.747481210Z INFO: 10.32.3.1:52820 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:01:50.730+00:00 2026-10-15T05:01:50.747013214Z INFO: 10.32.3.1:36308 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:02:00.734+00:00 2026-10-15T05:02:00.746899682Z INFO: 10.32.3.1:34568 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:02:10.729+00:00 2026-10-15T05:02:10.747400430Z INFO: 10.32.3.1:38018 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:02:20.729+00:00 2026-10-15T05:02:20.747204752Z INFO: 10.32.3.1:54932 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:02:26.583+00:00 2026-10-15T05:02:26.601098990Z INFO: 127.0.0.1:41122 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:02:30.730+00:00 2026-10-15T05:02:30.748457327Z INFO: 10.32.3.1:33506 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:02:40.729+00:00 2026-10-15T05:02:40.747125356Z INFO: 10.32.3.1:49354 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:02:50.733+00:00 2026-10-15T05:02:50.747387509Z INFO: 10.32.3.1:41164 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:03:00.728+00:00 2026-10-15T05:03:00.746612139Z INFO: 10.32.3.1:37988 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:03:10.728+00:00 2026-10-15T05:03:10.746728182Z INFO: 10.32.3.1:35818 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:03:16.776+00:00 2026-10-15T05:03:16.783574019Z INFO: 127.0.0.1:58382 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:03:20.727+00:00 2026-10-15T05:03:20.746403654Z INFO: 10.32.3.1:41964 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:03:30.821+00:00 2026-10-15T05:03:30.746892009Z INFO: 10.32.3.1:57042 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl new file mode 100644 index 00000000..15a11f70 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T05:01:15.634+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T05:01:15.634+00:00", "t": 0.001, "phase": "create_model.start"} +{"ts": "2026-10-15T05:01:31.419+00:00", "t": 15.785, "phase": "create_model.done", "model_id": "840431b4-6617-4074-8848-cf95526c9aec"} +{"ts": "2026-10-15T05:01:34.271+00:00", "t": 18.637, "phase": "forward_backward.start"} +{"ts": "2026-10-15T05:01:37.484+00:00", "t": 21.85, "phase": "forward_backward.done"} +{"ts": "2026-10-15T05:01:38.241+00:00", "t": 22.607, "phase": "optim_step.done"} +{"ts": "2026-10-15T05:01:38.245+00:00", "t": 22.611, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T05:01:38.818+00:00", "t": 23.184, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T05:01:38.819+00:00", "t": 23.185, "phase": "sample.start"} +{"ts": "2026-10-15T05:03:26.180+00:00", "t": 130.546, "phase": "sample.done", "text": " 4"} +{"ts": "2026-10-15T05:03:26.181+00:00", "t": 130.547, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log new file mode 100644 index 00000000..ee6051b8 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.001s create_model.start +[cold-start] 15.785s create_model.done {'model_id': '840431b4-6617-4074-8848-cf95526c9aec'} +[cold-start] 18.637s forward_backward.start +[cold-start] 21.850s forward_backward.done +[cold-start] 22.607s optim_step.done +[cold-start] 22.611s save_weights_and_get_sampling_client.start +[cold-start] 23.184s save_weights_and_get_sampling_client.done +[cold-start] 23.185s sample.start +[cold-start] 130.546s sample.done {'text': ' 4'} +[cold-start] 130.547s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt new file mode 100644 index 00000000..565ffbe3 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T05:03:26.774+00:00 diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json new file mode 100644 index 00000000..8ac2687a --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json @@ -0,0 +1,2025 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:10Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:10Z", + "name": "cold-start-fio.18de9a0ef295773d", + "namespace": "openrl-system", + "resourceVersion": "1792039090760383011", + "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5be63862", + "namespace": "openrl-system", + "resourceVersion": "1792039092515007000", + "uid": "355b59b9-3661-499d-8640-3f40c3c89375" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5ecfa51a", + "namespace": "openrl-system", + "resourceVersion": "1792039092561663021", + "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f61ddd1e4", + "namespace": "openrl-system", + "resourceVersion": "1792039092605935011", + "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:40:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:40:41Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:40:41Z", + "name": "cold-start-fio.18de9a32081cf908", + "namespace": "openrl-system", + "resourceVersion": "1792039241446287009", + "uid": "14a62e42-13ba-4526-a04d-a934702e1163" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:18Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:18Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:18Z", + "name": "cold-start-fio.18de9a3a9867ce9f", + "namespace": "openrl-system", + "resourceVersion": "1792039278208207001", + "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad2bcac91", + "namespace": "openrl-system", + "resourceVersion": "1792039279188751014", + "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad378cd74", + "namespace": "openrl-system", + "resourceVersion": "1792039279219679006", + "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad5aed221", + "namespace": "openrl-system", + "resourceVersion": "1792039279235135013", + "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:11Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:11Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:11Z", + "name": "cold-start-fio.18de9a70cd2c6a68", + "namespace": "openrl-system", + "resourceVersion": "1792039511048479001", + "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "resourceVersion": "1791941114537823021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Stopping container api-server", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", + "namespace": "openrl-system", + "resourceVersion": "1792039545418735013", + "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "namespace": "openrl-system", + "resourceVersion": "1792039545300351007", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", + "namespace": "openrl-system", + "resourceVersion": "1792039545387567001", + "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" + }, + "reason": "SuccessfulDelete", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546048591003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", + "namespace": "openrl-system", + "resourceVersion": "1792039546102351012", + "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:49Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:49Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", + "namespace": "openrl-system", + "resourceVersion": "1792039549712335019", + "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", + "namespace": "openrl-system", + "resourceVersion": "1792039558721215017", + "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", + "namespace": "openrl-system", + "resourceVersion": "1792039558783199013", + "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", + "namespace": "openrl-system", + "resourceVersion": "1792039558862303009", + "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "namespace": "openrl-system", + "resourceVersion": "1792039545940671018", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", + "namespace": "openrl-system", + "resourceVersion": "1792039546068975009", + "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" + }, + "reason": "SuccessfulCreate", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545232655018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78c9668cf7", + "namespace": "openrl-system", + "resourceVersion": "1792039545327119005", + "uid": "7b1433df-3524-478b-aba2-8979950e245e" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545464319018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78ef8b9e1b", + "namespace": "openrl-system", + "resourceVersion": "1792039545956079020", + "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:15Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875490623001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:15Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", + "namespace": "openrl-system", + "resourceVersion": "1792039875609279013", + "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:19Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:19Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", + "namespace": "openrl-system", + "resourceVersion": "1792039879126319008", + "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", + "namespace": "openrl-system", + "resourceVersion": "1792040061645247020", + "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", + "namespace": "openrl-system", + "resourceVersion": "1792040061849391002", + "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", + "namespace": "openrl-system", + "resourceVersion": "1792040062252079023", + "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", + "namespace": "openrl-system", + "resourceVersion": "1792040192780783021", + "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264403887001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", + "namespace": "openrl-system", + "resourceVersion": "1792040264444463021", + "uid": "a85006a2-68a6-4919-b148-08d58b546d07" + }, + "reason": "FailedScheduling", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Warning" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264434463001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", + "namespace": "openrl-system", + "resourceVersion": "1792040264884223002", + "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" + }, + "reason": "NotTriggerScaleUp", + "reportingComponent": "cluster-autoscaler", + "reportingInstance": "", + "source": { + "component": "cluster-autoscaler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264893039001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", + "namespace": "openrl-system", + "resourceVersion": "1792040293171567001", + "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", + "namespace": "openrl-system", + "resourceVersion": "1792040296672127014", + "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", + "namespace": "openrl-system", + "resourceVersion": "1792040296703007017", + "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", + "namespace": "openrl-system", + "resourceVersion": "1792040296889487014", + "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", + "namespace": "openrl-system", + "resourceVersion": "1792040427147711019", + "uid": "cc4da270-def2-456f-9374-372be8c717e4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:38Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498635999001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:38Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:38Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", + "namespace": "openrl-system", + "resourceVersion": "1792040498767471000", + "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", + "namespace": "openrl-system", + "resourceVersion": "1792040502270255017", + "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", + "namespace": "openrl-system", + "resourceVersion": "1792040502296943006", + "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", + "namespace": "openrl-system", + "resourceVersion": "1792040502475407023", + "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:52Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672220015012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:52Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", + "namespace": "openrl-system", + "resourceVersion": "1792039672350591019", + "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:56Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:56Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:56Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", + "namespace": "openrl-system", + "resourceVersion": "1792039676166239015", + "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", + "namespace": "openrl-system", + "resourceVersion": "1792039844457167000", + "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", + "namespace": "openrl-system", + "resourceVersion": "1792039844632591021", + "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:45Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:45Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", + "namespace": "openrl-system", + "resourceVersion": "1792039845170959016", + "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", + "namespace": "openrl-system", + "resourceVersion": "1792040192981183013", + "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:23Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040242926223012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:23Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:23Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", + "namespace": "openrl-system", + "resourceVersion": "1792040243043743024", + "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", + "namespace": "openrl-system", + "resourceVersion": "1792040246545487023", + "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", + "namespace": "openrl-system", + "resourceVersion": "1792040246576895006", + "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", + "namespace": "openrl-system", + "resourceVersion": "1792040246742815002", + "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", + "namespace": "openrl-system", + "resourceVersion": "1792040427285631023", + "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:16Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476805407012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:16Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", + "namespace": "openrl-system", + "resourceVersion": "1792040476942623014", + "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", + "namespace": "openrl-system", + "resourceVersion": "1792040480747055005", + "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", + "namespace": "openrl-system", + "resourceVersion": "1792040480787967019", + "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", + "namespace": "openrl-system", + "resourceVersion": "1792040480963103007", + "uid": "d3158913-c79e-43d4-b468-d748641d49fe" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", + "namespace": "openrl-system", + "resourceVersion": "1792038835732623016", + "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", + "namespace": "openrl-system", + "resourceVersion": "1792038835415695012", + "uid": "918abc9a-1f24-4767-8928-b202482f11ed" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt new file mode 100644 index 00000000..b57d24d4 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt @@ -0,0 +1,120 @@ +2026-10-15T05:02:05.308858557Z INFO 10-15 05:02:05 [model.py:692] Resolved architecture: Qwen2ForCausalLM +2026-10-15T05:02:05.309341375Z INFO 10-15 05:02:05 [model.py:2030] Using max model len 8192 +2026-10-15T05:02:05.311509464Z INFO 10-15 05:02:05 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. +2026-10-15T05:02:06.441297160Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T05:02:06.722078744Z INFO 10-15 05:02:06 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T05:02:10.058431917Z WARNING 10-15 05:02:10 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T05:02:21.575184973Z (EngineCore pid=161) INFO 10-15 05:02:21 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T05:02:22.390749815Z (EngineCore pid=161) INFO 10-15 05:02:22 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_da4da587630c4760931dd312e0d64a19 backend=nccl +2026-10-15T05:02:22.398295764Z (EngineCore pid=161) INFO 10-15 05:02:22 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T05:02:22.398324742Z (EngineCore pid=161) INFO 10-15 05:02:22 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T05:02:23.325863393Z (EngineCore pid=161) INFO 10-15 05:02:23 [model_runner.py:396] Loading model from scratch... +2026-10-15T05:02:24.203367263Z (EngineCore pid=161) INFO 10-15 05:02:24 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T05:02:24.203834642Z (EngineCore pid=161) INFO 10-15 05:02:24 [flash_attn.py:1116] Using FlashAttention version 2 +2026-10-15T05:02:24.320213947Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T05:02:24.320556376Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T05:02:24.679861647Z (EngineCore pid=161) INFO 10-15 05:02:24 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.61 GiB. +2026-10-15T05:02:24.679896304Z (EngineCore pid=161) INFO 10-15 05:02:24 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T05:02:24.681277221Z (EngineCore pid=161) +Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T05:01:27.867911082Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T05:01:27.867937281Z +2026-10-15T05:01:27.989644961Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... +2026-10-15T05:01:28.104973244Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T05:01:28.918184929Z [prefetch] reading 1 file(s), 0.92 GiB, from nfs into the page cache with 8 threads +2026-10-15T05:01:29.629438937Z [prefetch] done: 0.92 GiB in 0.7 s (1325 MiB/s) +2026-10-15T05:01:30.447285388Z +Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json new file mode 100644 index 00000000..5ee0198a --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "generateName": "open-rl-api-server-6d86b498f5-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "6d86b498f5", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + } + ], + "resourceVersion": "1792039570790271003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2ns79", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:59Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:45:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.11", + "podIPs": [ + { + "ip": "10.32.3.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:45:46Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:38Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "uid": "7b9b59dc-9b47-457a-ae6e-445216e97f77" + } + ], + "resourceVersion": "1792040502744719001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "dc12971c-f814-48f0-989f-61dcbd87544b" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2dmx2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2dmx2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:42Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "25Gi" + }, + "containerID": "containerd://9ab93d789300dabc0ddb29eecd8d52752a8620fb49c116df96da13be6a371073", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:01:42Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2dmx2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.0.22", + "podIPs": [ + { + "ip": "10.32.0.22" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:01:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:16Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", + "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", + "timeslice.io/owner": "qwen-qwen2-5-0-5b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "uid": "2e5fb99a-5dca-43b7-9297-8a45c4fc96e0" + } + ], + "resourceVersion": "1792040481706719012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "9ba5505c-ce23-4a59-ba01-41cad9093eca" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-g89d5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-g89d5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:21Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:16Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:21Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:21Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:01:16Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "21Gi" + }, + "containerID": "containerd://996f69bb4e5188384112befeb9a6e19076f01e6d3416b8e194129863978b175f", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:01:20Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-g89d5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.13.28", + "podIPs": [ + { + "ip": "10.32.13.28" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:01:16Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log new file mode 100644 index 00000000..62cf050e --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log @@ -0,0 +1,153 @@ +2026-10-15T05:01:12.427+00:00 2026-10-15T05:01:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:13.431+00:00 2026-10-15T05:01:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:14.437+00:00 2026-10-15T05:01:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:15.442+00:00 2026-10-15T05:01:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:16.446+00:00 2026-10-15T05:01:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:17.451+00:00 2026-10-15T05:01:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:18.458+00:00 2026-10-15T05:01:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:19.467+00:00 2026-10-15T05:01:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:20.468+00:00 2026-10-15T05:01:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:21.477+00:00 2026-10-15T05:01:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:22.478+00:00 2026-10-15T05:01:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:23.483+00:00 2026-10-15T05:01:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:24.489+00:00 2026-10-15T05:01:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:25.494+00:00 2026-10-15T05:01:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:26.508+00:00 2026-10-15T05:01:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:27.503+00:00 2026-10-15T05:01:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:28.508+00:00 2026-10-15T05:01:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:29.516+00:00 2026-10-15T05:01:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:30.519+00:00 2026-10-15T05:01:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:31.524+00:00 2026-10-15T05:01:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:32.529+00:00 2026-10-15T05:01:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:33.534+00:00 2026-10-15T05:01:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:34.539+00:00 2026-10-15T05:01:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:35.545+00:00 2026-10-15T05:01:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:36.550+00:00 2026-10-15T05:01:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:37.555+00:00 2026-10-15T05:01:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:38.560+00:00 2026-10-15T05:01:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:39.565+00:00 2026-10-15T05:01:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:40.574+00:00 2026-10-15T05:01:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:41.576+00:00 2026-10-15T05:01:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:42.581+00:00 2026-10-15T05:01:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:43.588+00:00 2026-10-15T05:01:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:44.592+00:00 2026-10-15T05:01:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:45.598+00:00 2026-10-15T05:01:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:46.602+00:00 2026-10-15T05:01:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:47.608+00:00 2026-10-15T05:01:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:48.613+00:00 2026-10-15T05:01:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:49.618+00:00 2026-10-15T05:01:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:50.624+00:00 2026-10-15T05:01:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:51.628+00:00 2026-10-15T05:01:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:52.644+00:00 2026-10-15T05:01:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:53.654+00:00 2026-10-15T05:01:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:54.650+00:00 2026-10-15T05:01:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:55.648+00:00 2026-10-15T05:01:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:56.653+00:00 2026-10-15T05:01:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:57.659+00:00 2026-10-15T05:01:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:58.673+00:00 2026-10-15T05:01:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:01:59.674+00:00 2026-10-15T05:01:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:00.675+00:00 2026-10-15T05:02:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:01.679+00:00 2026-10-15T05:02:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:02.687+00:00 2026-10-15T05:02:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:03.690+00:00 2026-10-15T05:02:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:04.695+00:00 2026-10-15T05:02:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:05.700+00:00 2026-10-15T05:02:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:06.706+00:00 2026-10-15T05:02:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:07.710+00:00 2026-10-15T05:02:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:08.720+00:00 2026-10-15T05:02:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:09.721+00:00 2026-10-15T05:02:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:10.726+00:00 2026-10-15T05:02:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:11.731+00:00 2026-10-15T05:02:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:12.736+00:00 2026-10-15T05:02:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:13.767+00:00 2026-10-15T05:02:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:14.746+00:00 2026-10-15T05:02:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:15.752+00:00 2026-10-15T05:02:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:16.758+00:00 2026-10-15T05:02:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:17.764+00:00 2026-10-15T05:02:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:18.780+00:00 2026-10-15T05:02:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:19.773+00:00 2026-10-15T05:02:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:20.777+00:00 2026-10-15T05:02:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:21.783+00:00 2026-10-15T05:02:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:22.790+00:00 2026-10-15T05:02:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:23.794+00:00 2026-10-15T05:02:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:24.817+00:00 2026-10-15T05:02:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:25.805+00:00 2026-10-15T05:02:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:26.809+00:00 2026-10-15T05:02:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:27.815+00:00 2026-10-15T05:02:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:28.820+00:00 2026-10-15T05:02:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:29.825+00:00 2026-10-15T05:02:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:30.831+00:00 2026-10-15T05:02:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:31.836+00:00 2026-10-15T05:02:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:32.845+00:00 2026-10-15T05:02:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:33.847+00:00 2026-10-15T05:02:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:34.852+00:00 2026-10-15T05:02:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:35.865+00:00 2026-10-15T05:02:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:36.874+00:00 2026-10-15T05:02:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:37.867+00:00 2026-10-15T05:02:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:38.875+00:00 2026-10-15T05:02:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:39.878+00:00 2026-10-15T05:02:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:40.883+00:00 2026-10-15T05:02:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:41.888+00:00 2026-10-15T05:02:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:42.893+00:00 2026-10-15T05:02:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:43.898+00:00 2026-10-15T05:02:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:44.905+00:00 2026-10-15T05:02:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:45.916+00:00 2026-10-15T05:02:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:46.915+00:00 2026-10-15T05:02:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:47.919+00:00 2026-10-15T05:02:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:48.925+00:00 2026-10-15T05:02:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:49.930+00:00 2026-10-15T05:02:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:50.935+00:00 2026-10-15T05:02:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:51.939+00:00 2026-10-15T05:02:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:52.944+00:00 2026-10-15T05:02:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:53.949+00:00 2026-10-15T05:02:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:54.955+00:00 2026-10-15T05:02:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:55.959+00:00 2026-10-15T05:02:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:56.966+00:00 2026-10-15T05:02:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:57.973+00:00 2026-10-15T05:02:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:58.975+00:00 2026-10-15T05:02:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:02:59.981+00:00 2026-10-15T05:02:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:00.987+00:00 2026-10-15T05:03:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:01.991+00:00 2026-10-15T05:03:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:02.997+00:00 2026-10-15T05:03:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:04.005+00:00 2026-10-15T05:03:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:05.011+00:00 2026-10-15T05:03:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:06.012+00:00 2026-10-15T05:03:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:07.020+00:00 2026-10-15T05:03:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:08.022+00:00 2026-10-15T05:03:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:09.027+00:00 2026-10-15T05:03:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:10.034+00:00 2026-10-15T05:03:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:11.038+00:00 2026-10-15T05:03:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:12.043+00:00 2026-10-15T05:03:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:13.048+00:00 2026-10-15T05:03:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:14.055+00:00 2026-10-15T05:03:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:15.059+00:00 2026-10-15T05:03:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:16.067+00:00 2026-10-15T05:03:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:17.069+00:00 2026-10-15T05:03:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:18.078+00:00 2026-10-15T05:03:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:19.079+00:00 2026-10-15T05:03:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:20.085+00:00 2026-10-15T05:03:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:21.093+00:00 2026-10-15T05:03:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:22.097+00:00 2026-10-15T05:03:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:23.100+00:00 2026-10-15T05:03:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:24.105+00:00 2026-10-15T05:03:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:25.110+00:00 2026-10-15T05:03:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:26.123+00:00 2026-10-15T05:03:26Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:26.123+00:00 2026-10-15T05:03:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:27.142+00:00 2026-10-15T05:03:27Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:27.142+00:00 2026-10-15T05:03:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:28.126+00:00 2026-10-15T05:03:28Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:28.126+00:00 2026-10-15T05:03:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:29.130+00:00 2026-10-15T05:03:29Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:29.130+00:00 2026-10-15T05:03:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:30.157+00:00 2026-10-15T05:03:30Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:30.157+00:00 2026-10-15T05:03:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:31.216+00:00 2026-10-15T05:03:31Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:31.216+00:00 2026-10-15T05:03:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:32.160+00:00 2026-10-15T05:03:32Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:32.160+00:00 2026-10-15T05:03:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:33.185+00:00 2026-10-15T05:03:33Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:33.186+00:00 2026-10-15T05:03:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:34.183+00:00 2026-10-15T05:03:34Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:34.183+00:00 2026-10-15T05:03:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:35.226+00:00 2026-10-15T05:03:35Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:03:35.226+00:00 2026-10-15T05:03:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log new file mode 100644 index 00000000..0a19fdc3 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T05:01:16.606+00:00 2026-10-15T05:01:16.624535671Z {"level":"info","ts":"2026-10-15T05:01:16Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"9572772f-a156-403a-b664-332350bac2f5","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} +2026-10-15T05:01:38.487+00:00 2026-10-15T05:01:38.505401957Z {"level":"info","ts":"2026-10-15T05:01:38Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"1a58b154-9236-4c92-aab9-519ccf1bd949","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json new file mode 100644 index 00000000..05335083 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json @@ -0,0 +1,267 @@ +{ + "case": "after-B2-0.5b-l4-cached", + "collected": "2026-10-15T05:03:35.925+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 05:01:15.634000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen2.5-0.5B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 05:01:15.634000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 05:01:31.419000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "840431b4-6617-4074-8848-cf95526c9aec" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 05:01:34.271000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 05:01:37.484000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 05:01:38.241000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 05:01:38.245000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 05:01:38.818000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 05:01:38.819000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 05:03:26.180000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4" + }, + "client.end": { + "ts": "2026-10-15 05:03:26.181000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 05:01:38+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:01:38.028000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 05:01:38.028000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 05:01:39.789000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 05:01:42+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 05:01:38+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 05:01:38+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 05:01:42+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 05:01:42+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 05:01:38+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 05:01:42+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 05:01:38+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 05:01:42+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "sampler.event_Created": { + "ts": "2026-10-15 05:01:42+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 05:01:42+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 05:02:05.308858+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 05:02:21.575184+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:21 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 05:02:23.325863+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:23 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 05:02:24.679896+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:24 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 05:02:25.907445+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:25 [default_loader.py:430] Loading weights took 1.23 seconds", + "self_reported_s": 1.23 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 05:02:26.718502+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:26 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.717496 seconds", + "self_reported_s": 3.717496 + }, + "sampler.compile_done": { + "ts": "2026-10-15 05:02:57.251696+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:57 [monitor.py:53] torch.compile took 28.32 s in total", + "self_reported_s": 28.32 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 05:03:12.777714+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:03:12 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", + "self_reported_s": 11.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 05:03:13.327483+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:03:13 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 05:03:24.097775+00:00", + "source": "container log: (EngineCore pid=161) INFO 10-15 05:03:24 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.37 s", + "self_reported_s": 57.37 + }, + "sampler.ready_key": { + "ts": "2026-10-15 05:03:26+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + }, + "trainer": { + "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", + "model_id": "Qwen/Qwen2.5-0.5B", + "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 05:01:16+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:01:16.282000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 05:01:16.282000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 05:01:17.948000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 05:01:22+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 05:01:16+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 05:01:16+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 05:01:21+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 05:01:21+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 05:01:16+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 05:01:20+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 05:01:16+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 05:01:20+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "trainer.event_Created": { + "ts": "2026-10-15 05:01:20+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 05:01:20+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 05:01:27.847766+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 05:01:27.989644+00:00", + "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 05:01:30.531288+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 05:01:30.533708+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 05:01:12+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.md b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.md new file mode 100644 index 00000000..6c550078 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.md @@ -0,0 +1,109 @@ +# Cold start timeline: after-B2-0.5b-l4-cached + +t=0 is the client's `create_model` call at 2026-10-15T05:01:15.634+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen2-5-0-5b-0-trainer` on `gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.ready_key | 05:01:12.000 | -3.6 | | redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B | +| trainer.workload_created | 05:01:16.000 | 0.4 | | Workload creationTimestamp (apiserver) | +| trainer.pod_created | 05:01:16.000 | 0.4 | | pod creationTimestamp (apiserver) | +| trainer.pod_Initialized | 05:01:16.000 | 0.4 | | pod condition Initialized | +| trainer.pod_PodScheduled | 05:01:16.000 | 0.4 | | pod condition PodScheduled | +| trainer.event_Scheduled | 05:01:16.000 | 0.4 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 | +| trainer.workload_first_seen_by_poller | 05:01:16.282 | 0.6 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 05:01:16.282 | 0.6 | | workloads.tsv poller: status.podName set | +| trainer.workload_reports_node | 05:01:17.948 | 2.3 | | workloads.tsv poller: status.nodeName set | +| trainer.container_started | 05:01:20.000 | 4.4 | | containerStatuses.state.running.startedAt | +| trainer.event_Pulled | 05:01:20.000 | 4.4 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| trainer.event_Created | 05:01:20.000 | 4.4 | | event: Container created | +| trainer.event_Started | 05:01:20.000 | 4.4 | | event: Container started | +| trainer.pod_Ready | 05:01:21.000 | 5.4 | | pod condition Ready | +| trainer.pod_ContainersReady | 05:01:21.000 | 5.4 | | pod condition ContainersReady | +| trainer.condition_Placed_True | 05:01:22.000 | 6.4 | | Workload condition Placed | +| trainer.process_first_log | 05:01:27.847 | 12.2 | | container log, first line | +| trainer.load_start | 05:01:27.989 | 12.4 | | container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0... | +| trainer.load_done | 05:01:30.531 | 14.9 | | container log: Successfully loaded. | +| trainer.processor_started | 05:01:30.533 | 14.9 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1). | + +## sampler: `orw-lora-qwen-qwen2-5-0-5b-0-sampler` on `gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_created | 05:01:38.000 | 22.4 | | Workload creationTimestamp (apiserver) | +| sampler.pod_created | 05:01:38.000 | 22.4 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 05:01:38.000 | 22.4 | | pod condition Initialized | +| sampler.pod_PodScheduled | 05:01:38.000 | 22.4 | | pod condition PodScheduled | +| sampler.event_Scheduled | 05:01:38.000 | 22.4 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 | +| sampler.workload_first_seen_by_poller | 05:01:38.028 | 22.4 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_reports_pod | 05:01:38.028 | 22.4 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 05:01:39.789 | 24.2 | | workloads.tsv poller: status.nodeName set | +| sampler.condition_Placed_True | 05:01:42.000 | 26.4 | | Workload condition Placed | +| sampler.pod_Ready | 05:01:42.000 | 26.4 | | pod condition Ready | +| sampler.pod_ContainersReady | 05:01:42.000 | 26.4 | | pod condition ContainersReady | +| sampler.container_started | 05:01:42.000 | 26.4 | | containerStatuses.state.running.startedAt | +| sampler.event_Pulled | 05:01:42.000 | 26.4 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| sampler.event_Created | 05:01:42.000 | 26.4 | | event: Container created | +| sampler.event_Started | 05:01:42.000 | 26.4 | | event: Container started | +| sampler.process_first_log | 05:02:05.308 | 49.7 | | container log, first line | +| sampler.engine_init_start | 05:02:21.575 | 65.9 | | container log: (EngineCore pid=161) INFO 10-15 05:02:21 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 05:02:23.325 | 67.7 | | container log: (EngineCore pid=161) INFO 10-15 05:02:23 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 05:02:24.679 | 69.0 | | container log: (EngineCore pid=161) INFO 10-15 05:02:24 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 05:02:25.907 | 70.3 | 1.23 | container log: (EngineCore pid=161) INFO 10-15 05:02:25 [default_loader.py:430] Loading weights took 1.23 seconds | +| sampler.model_loaded | 05:02:26.718 | 71.1 | 3.717496 | container log: (EngineCore pid=161) INFO 10-15 05:02:26 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.717496 seconds | +| sampler.compile_done | 05:02:57.251 | 101.6 | 28.32 | container log: (EngineCore pid=161) INFO 10-15 05:02:57 [monitor.py:53] torch.compile took 28.32 s in total | +| sampler.graphs_captured | 05:03:12.777 | 117.1 | 11.0 | container log: (EngineCore pid=161) INFO 10-15 05:03:12 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB | +| sampler.kv_cache_sized | 05:03:13.327 | 117.7 | | container log: (EngineCore pid=161) INFO 10-15 05:03:13 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB | +| sampler.engine_init_done | 05:03:24.097 | 128.5 | 57.37 | container log: (EngineCore pid=161) INFO 10-15 05:03:24 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.37 s | +| sampler.ready_key | 05:03:26.000 | 130.4 | | redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 05:01:15.634 | 0.0 | +| client.create_model.start | 05:01:15.634 | 0.0 | +| client.create_model.done | 05:01:31.419 | 15.8 | +| client.forward_backward.start | 05:01:34.271 | 18.6 | +| client.forward_backward.done | 05:01:37.484 | 21.9 | +| client.optim_step.done | 05:01:38.241 | 22.6 | +| client.save_weights_and_get_sampling_client.start | 05:01:38.245 | 22.6 | +| client.save_weights_and_get_sampling_client.done | 05:01:38.818 | 23.2 | +| client.sample.start | 05:01:38.819 | 23.2 | +| client.sample.done | 05:03:26.180 | 130.5 | +| client.end | 05:03:26.181 | 130.5 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.4 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 8.0 | +| weights load | trainer.load_start | trainer.load_done | 2.5 | +| ready key | trainer.load_done | trainer.ready_key | -18.5 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 15.8 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.2 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 23.3 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 16.3 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.8 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.6 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.3 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.8 | +| ready key | sampler.engine_init_done | sampler.ready_key | 1.9 | +| client: first sample blocked | client.sample.start | client.sample.done | 107.4 | + diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt new file mode 100644 index 00000000..b19e78b5 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T05:01:11.324+00:00 diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt new file mode 100644 index 00000000..a9917932 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json new file mode 100644 index 00000000..a4b62ba3 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:38Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040502944559001", + "uid": "7b9b59dc-9b47-457a-ae6e-445216e97f77" + }, + "spec": { + "accelerator": { + "memory": "4Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "3589169104" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "25Gi" + }, + "requests": { + "memory": "25Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "dc12971c-f814-48f0-989f-61dcbd87544b", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:01:42Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "4Gi", + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:16Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040482420783015", + "uid": "2e5fb99a-5dca-43b7-9297-8a45c4fc96e0" + }, + "spec": { + "accelerator": { + "memory": "5Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen2.5-0.5B", + "ownerID": "qwen-qwen2-5-0-5b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen2.5-0.5B", + "--active-tenant-set-id", + "Qwen/Qwen2.5-0.5B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen2.5-0.5B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "5282967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen2-5-0-5b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "21Gi" + }, + "requests": { + "memory": "21Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "9ba5505c-ce23-4a59-ba01-41cad9093eca", + "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:01:22Z", + "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "5Gi", + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log new file mode 100644 index 00000000..afa62d96 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log @@ -0,0 +1,35 @@ +watching; recorders [16726, 16727, 16729, 16731, 16733, 16728, 16730, 16732] write into runs/2026-10-15_cold_start/after-C1-8b-h100-cold +collected 2 worker log(s) into runs/2026-10-15_cold_start/after-C1-8b-h100-cold +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.9 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 90.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.1 | +| weights load | trainer.load_start | trainer.load_done | 13.0 | +| ready key | trainer.load_done | trainer.ready_key | -0.1 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 111.5 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.8 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.4 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 90.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 7.5 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 17.5 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 14.0 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.5 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.9 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.6 | +| client: first sample blocked | client.sample.start | client.sample.done | 207.9 | + diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log new file mode 100644 index 00000000..4097aea9 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log @@ -0,0 +1,56 @@ +2026-10-15T05:04:58.373+00:00 2026-10-15T05:04:58.392830523Z INFO: 127.0.0.1:49988 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T05:04:58.599+00:00 2026-10-15T05:04:58.617434754Z INFO: 127.0.0.1:49998 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T05:04:58.671+00:00 2026-10-15T05:04:58.691762251Z INFO: 127.0.0.1:49998 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T05:04:58.912+00:00 2026-10-15T05:04:58.933022406Z [API_SERVER] enqueued op=create_model request_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:04:58.912+00:00 2026-10-15T05:04:58.934353835Z INFO: 127.0.0.1:50002 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T05:05:00.726+00:00 2026-10-15T05:05:00.747482128Z INFO: 10.32.3.1:57206 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:05:08.868+00:00 2026-10-15T05:05:08.886137925Z INFO: 127.0.0.1:46950 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:05:10.727+00:00 2026-10-15T05:05:10.747267963Z INFO: 10.32.3.1:60820 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:05:20.728+00:00 2026-10-15T05:05:20.746735897Z INFO: 10.32.3.1:45172 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:05:30.730+00:00 2026-10-15T05:05:30.747739235Z INFO: 10.32.3.1:42562 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:05:32.132+00:00 2026-10-15T05:05:32.152552067Z [API_SERVER] No live session uses qwen-qwen2-5-0-5b; tearing its workers down +2026-10-15T05:05:40.729+00:00 2026-10-15T05:05:40.748089335Z INFO: 10.32.3.1:39376 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:05:49.048+00:00 2026-10-15T05:05:49.066216479Z INFO: 127.0.0.1:38418 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:05:50.725+00:00 2026-10-15T05:05:50.747579061Z INFO: 10.32.3.1:46244 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:06:00.730+00:00 2026-10-15T05:06:00.747988598Z INFO: 10.32.3.1:44906 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:06:10.731+00:00 2026-10-15T05:06:10.748609740Z INFO: 10.32.3.1:46228 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:06:20.729+00:00 2026-10-15T05:06:20.747688850Z INFO: 10.32.3.1:60052 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:06:30.728+00:00 2026-10-15T05:06:30.747502924Z INFO: 10.32.3.1:43338 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:06:39.279+00:00 2026-10-15T05:06:39.296775538Z INFO: 127.0.0.1:47514 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:06:40.724+00:00 2026-10-15T05:06:40.747079682Z INFO: 10.32.3.1:54366 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:06:49.774+00:00 2026-10-15T05:06:49.794105686Z INFO: 127.0.0.1:40780 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T05:06:50.724+00:00 2026-10-15T05:06:50.746682232Z INFO: 10.32.3.1:54610 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:06:52.856+00:00 2026-10-15T05:06:52.877671436Z [API_SERVER] enqueued op=forward_backward request_id=89817a3b-47fd-4fe7-ac4f-96f85e67f00d model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:06:52.857+00:00 2026-10-15T05:06:52.879485463Z INFO: 127.0.0.1:40780 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T05:06:55.673+00:00 2026-10-15T05:06:55.694412936Z [API_SERVER] enqueued op=optim_step request_id=960757cf-ac36-466e-8ac6-4739735165f9 model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:06:55.676+00:00 2026-10-15T05:06:55.695623233Z INFO: 127.0.0.1:40780 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T05:06:57.720+00:00 2026-10-15T05:06:57.740198942Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=8e6d158e-97d2-4b55-969a-3c06dfb5aaf1 model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:06:57.720+00:00 2026-10-15T05:06:57.741174270Z INFO: 127.0.0.1:40780 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T05:06:59.833+00:00 2026-10-15T05:06:59.854047429Z INFO: 127.0.0.1:38466 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T05:07:00.725+00:00 2026-10-15T05:07:00.748423093Z INFO: 10.32.3.1:34800 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:07:10.729+00:00 2026-10-15T05:07:10.747610422Z INFO: 10.32.3.1:37484 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:07:20.727+00:00 2026-10-15T05:07:20.747378396Z INFO: 10.32.3.1:46200 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:07:30.729+00:00 2026-10-15T05:07:30.746887455Z INFO: 10.32.3.1:32866 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:07:40.729+00:00 2026-10-15T05:07:40.747803363Z INFO: 10.32.3.1:34618 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:07:49.470+00:00 2026-10-15T05:07:49.486856420Z INFO: 127.0.0.1:35182 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:07:50.727+00:00 2026-10-15T05:07:50.749079833Z INFO: 10.32.3.1:44656 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:08:00.731+00:00 2026-10-15T05:08:00.746828463Z INFO: 10.32.3.1:52534 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:08:10.729+00:00 2026-10-15T05:08:10.747693054Z INFO: 10.32.3.1:45248 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:08:20.741+00:00 2026-10-15T05:08:20.747102416Z INFO: 10.32.3.1:35372 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:08:30.727+00:00 2026-10-15T05:08:30.747138544Z INFO: 10.32.3.1:46088 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:08:39.651+00:00 2026-10-15T05:08:39.669833714Z INFO: 127.0.0.1:42494 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:08:40.820+00:00 2026-10-15T05:08:40.748143407Z INFO: 10.32.3.1:60424 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:08:50.737+00:00 2026-10-15T05:08:50.748661295Z INFO: 10.32.3.1:34518 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:09:00.727+00:00 2026-10-15T05:09:00.747876484Z INFO: 10.32.3.1:50858 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:09:10.728+00:00 2026-10-15T05:09:10.748210445Z INFO: 10.32.3.1:35368 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:09:19.829+00:00 2026-10-15T05:09:19.849115515Z INFO: 127.0.0.1:47558 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:09:20.723+00:00 2026-10-15T05:09:20.746874486Z INFO: 10.32.3.1:39832 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:09:30.729+00:00 2026-10-15T05:09:30.747161062Z INFO: 10.32.3.1:47028 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:09:40.727+00:00 2026-10-15T05:09:40.747722414Z INFO: 10.32.3.1:41054 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:09:50.725+00:00 2026-10-15T05:09:50.746880809Z INFO: 10.32.3.1:57554 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:09:58.736+00:00 2026-10-15T05:09:58.758631423Z INFO: 127.0.0.1:49698 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T05:10:00.726+00:00 2026-10-15T05:10:00.747754462Z INFO: 10.32.3.1:51350 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:10:10.039+00:00 2026-10-15T05:10:10.061264177Z INFO: 127.0.0.1:42950 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:10:10.723+00:00 2026-10-15T05:10:10.747463025Z INFO: 10.32.3.1:46708 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:10:20.725+00:00 2026-10-15T05:10:20.746995461Z INFO: 10.32.3.1:50678 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:10:30.725+00:00 2026-10-15T05:10:30.747017967Z INFO: 10.32.3.1:36428 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl new file mode 100644 index 00000000..a66ee2ba --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T05:04:58.138+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T05:04:58.139+00:00", "t": 0.0, "phase": "create_model.start"} +{"ts": "2026-10-15T05:06:49.594+00:00", "t": 111.456, "phase": "create_model.done", "model_id": "048101ca-2c60-4c63-afa2-2ae75e83f1c4"} +{"ts": "2026-10-15T05:06:52.784+00:00", "t": 114.646, "phase": "forward_backward.start"} +{"ts": "2026-10-15T05:06:55.595+00:00", "t": 117.457, "phase": "forward_backward.done"} +{"ts": "2026-10-15T05:06:57.617+00:00", "t": 119.479, "phase": "optim_step.done"} +{"ts": "2026-10-15T05:06:57.618+00:00", "t": 119.48, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T05:06:59.533+00:00", "t": 121.395, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T05:06:59.534+00:00", "t": 121.395, "phase": "sample.start"} +{"ts": "2026-10-15T05:10:27.431+00:00", "t": 329.293, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} +{"ts": "2026-10-15T05:10:27.434+00:00", "t": 329.296, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log new file mode 100644 index 00000000..5a4ebab2 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.000s create_model.start +[cold-start] 111.456s create_model.done {'model_id': '048101ca-2c60-4c63-afa2-2ae75e83f1c4'} +[cold-start] 114.646s forward_backward.start +[cold-start] 117.457s forward_backward.done +[cold-start] 119.479s optim_step.done +[cold-start] 119.480s save_weights_and_get_sampling_client.start +[cold-start] 121.395s save_weights_and_get_sampling_client.done +[cold-start] 121.395s sample.start +[cold-start] 329.293s sample.done {'text': ' 4\n\nQuestion: What is '} +[cold-start] 329.296s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt new file mode 100644 index 00000000..8b3c4311 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T05:10:28.035+00:00 diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json new file mode 100644 index 00000000..89cb0ffe --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json @@ -0,0 +1,2417 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:10Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:10Z", + "name": "cold-start-fio.18de9a0ef295773d", + "namespace": "openrl-system", + "resourceVersion": "1792039090760383011", + "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5be63862", + "namespace": "openrl-system", + "resourceVersion": "1792039092515007000", + "uid": "355b59b9-3661-499d-8640-3f40c3c89375" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5ecfa51a", + "namespace": "openrl-system", + "resourceVersion": "1792039092561663021", + "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f61ddd1e4", + "namespace": "openrl-system", + "resourceVersion": "1792039092605935011", + "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:40:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:40:41Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:40:41Z", + "name": "cold-start-fio.18de9a32081cf908", + "namespace": "openrl-system", + "resourceVersion": "1792039241446287009", + "uid": "14a62e42-13ba-4526-a04d-a934702e1163" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:18Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:18Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:18Z", + "name": "cold-start-fio.18de9a3a9867ce9f", + "namespace": "openrl-system", + "resourceVersion": "1792039278208207001", + "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad2bcac91", + "namespace": "openrl-system", + "resourceVersion": "1792039279188751014", + "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad378cd74", + "namespace": "openrl-system", + "resourceVersion": "1792039279219679006", + "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad5aed221", + "namespace": "openrl-system", + "resourceVersion": "1792039279235135013", + "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:11Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:11Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:11Z", + "name": "cold-start-fio.18de9a70cd2c6a68", + "namespace": "openrl-system", + "resourceVersion": "1792039511048479001", + "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "resourceVersion": "1791941114537823021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Stopping container api-server", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", + "namespace": "openrl-system", + "resourceVersion": "1792039545418735013", + "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "namespace": "openrl-system", + "resourceVersion": "1792039545300351007", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", + "namespace": "openrl-system", + "resourceVersion": "1792039545387567001", + "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" + }, + "reason": "SuccessfulDelete", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546048591003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", + "namespace": "openrl-system", + "resourceVersion": "1792039546102351012", + "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:49Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:49Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", + "namespace": "openrl-system", + "resourceVersion": "1792039549712335019", + "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", + "namespace": "openrl-system", + "resourceVersion": "1792039558721215017", + "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", + "namespace": "openrl-system", + "resourceVersion": "1792039558783199013", + "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", + "namespace": "openrl-system", + "resourceVersion": "1792039558862303009", + "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "namespace": "openrl-system", + "resourceVersion": "1792039545940671018", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", + "namespace": "openrl-system", + "resourceVersion": "1792039546068975009", + "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" + }, + "reason": "SuccessfulCreate", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545232655018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78c9668cf7", + "namespace": "openrl-system", + "resourceVersion": "1792039545327119005", + "uid": "7b1433df-3524-478b-aba2-8979950e245e" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545464319018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78ef8b9e1b", + "namespace": "openrl-system", + "resourceVersion": "1792039545956079020", + "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:15Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875490623001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:15Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", + "namespace": "openrl-system", + "resourceVersion": "1792039875609279013", + "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:19Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:19Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", + "namespace": "openrl-system", + "resourceVersion": "1792039879126319008", + "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", + "namespace": "openrl-system", + "resourceVersion": "1792040061645247020", + "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", + "namespace": "openrl-system", + "resourceVersion": "1792040061849391002", + "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", + "namespace": "openrl-system", + "resourceVersion": "1792040062252079023", + "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", + "namespace": "openrl-system", + "resourceVersion": "1792040192780783021", + "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264403887001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", + "namespace": "openrl-system", + "resourceVersion": "1792040264444463021", + "uid": "a85006a2-68a6-4919-b148-08d58b546d07" + }, + "reason": "FailedScheduling", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Warning" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264434463001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", + "namespace": "openrl-system", + "resourceVersion": "1792040264884223002", + "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" + }, + "reason": "NotTriggerScaleUp", + "reportingComponent": "cluster-autoscaler", + "reportingInstance": "", + "source": { + "component": "cluster-autoscaler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264893039001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", + "namespace": "openrl-system", + "resourceVersion": "1792040293171567001", + "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", + "namespace": "openrl-system", + "resourceVersion": "1792040296672127014", + "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", + "namespace": "openrl-system", + "resourceVersion": "1792040296703007017", + "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", + "namespace": "openrl-system", + "resourceVersion": "1792040296889487014", + "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", + "namespace": "openrl-system", + "resourceVersion": "1792040427147711019", + "uid": "cc4da270-def2-456f-9374-372be8c717e4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:38Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498635999001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:38Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:38Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", + "namespace": "openrl-system", + "resourceVersion": "1792040498767471000", + "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", + "namespace": "openrl-system", + "resourceVersion": "1792040502270255017", + "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", + "namespace": "openrl-system", + "resourceVersion": "1792040502296943006", + "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", + "namespace": "openrl-system", + "resourceVersion": "1792040502475407023", + "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:09Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:09Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:09Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", + "namespace": "openrl-system", + "resourceVersion": "1792040649924575012", + "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:52Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672220015012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:52Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", + "namespace": "openrl-system", + "resourceVersion": "1792039672350591019", + "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:56Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:56Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:56Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", + "namespace": "openrl-system", + "resourceVersion": "1792039676166239015", + "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", + "namespace": "openrl-system", + "resourceVersion": "1792039844457167000", + "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", + "namespace": "openrl-system", + "resourceVersion": "1792039844632591021", + "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:45Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:45Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", + "namespace": "openrl-system", + "resourceVersion": "1792039845170959016", + "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", + "namespace": "openrl-system", + "resourceVersion": "1792040192981183013", + "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:23Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040242926223012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:23Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:23Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", + "namespace": "openrl-system", + "resourceVersion": "1792040243043743024", + "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", + "namespace": "openrl-system", + "resourceVersion": "1792040246545487023", + "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", + "namespace": "openrl-system", + "resourceVersion": "1792040246576895006", + "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", + "namespace": "openrl-system", + "resourceVersion": "1792040246742815002", + "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", + "namespace": "openrl-system", + "resourceVersion": "1792040427285631023", + "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:16Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476805407012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:16Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", + "namespace": "openrl-system", + "resourceVersion": "1792040476942623014", + "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", + "namespace": "openrl-system", + "resourceVersion": "1792040480747055005", + "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", + "namespace": "openrl-system", + "resourceVersion": "1792040480787967019", + "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", + "namespace": "openrl-system", + "resourceVersion": "1792040480963103007", + "uid": "d3158913-c79e-43d4-b468-d748641d49fe" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:10Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:10Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", + "namespace": "openrl-system", + "resourceVersion": "1792040650191935010", + "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", + "namespace": "openrl-system", + "resourceVersion": "1792038835732623016", + "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:58Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818213471023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:58Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:58Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", + "namespace": "openrl-system", + "resourceVersion": "1792040818411407005", + "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:07:01Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:07:01Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:07:01Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", + "namespace": "openrl-system", + "resourceVersion": "1792040821861695012", + "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", + "namespace": "openrl-system", + "resourceVersion": "1792040908347887024", + "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", + "namespace": "openrl-system", + "resourceVersion": "1792040908367439012", + "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", + "namespace": "openrl-system", + "resourceVersion": "1792040908418671004", + "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", + "namespace": "openrl-system", + "resourceVersion": "1792038835415695012", + "uid": "918abc9a-1f24-4767-8928-b202482f11ed" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:59Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699253647006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:59Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:59Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", + "namespace": "openrl-system", + "resourceVersion": "1792040699411615020", + "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:05:02Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:05:02Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:05:02Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", + "namespace": "openrl-system", + "resourceVersion": "1792040702849391020", + "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", + "namespace": "openrl-system", + "resourceVersion": "1792040789347727015", + "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", + "namespace": "openrl-system", + "resourceVersion": "1792040789380863020", + "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", + "namespace": "openrl-system", + "resourceVersion": "1792040789417615006", + "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt new file mode 100644 index 00000000..c45083ae --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt @@ -0,0 +1,223 @@ +2026-10-15T05:08:35.508944913Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T05:08:43.408014705Z INFO 10-15 05:08:43 [model.py:692] Resolved architecture: Qwen3ForCausalLM +2026-10-15T05:08:43.408258811Z INFO 10-15 05:08:43 [model.py:2030] Using max model len 8192 +2026-10-15T05:08:43.409800014Z INFO 10-15 05:08:43 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. +2026-10-15T05:08:43.410611953Z INFO 10-15 05:08:43 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T05:08:45.859619092Z WARNING 10-15 05:08:45 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T05:08:52.969476089Z (EngineCore pid=169) INFO 10-15 05:08:52 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T05:08:53.310496973Z (EngineCore pid=169) INFO 10-15 05:08:53 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_fbe13a8f695f4e8c9b96a62df6e018fb backend=nccl +2026-10-15T05:08:53.314764503Z (EngineCore pid=169) INFO 10-15 05:08:53 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T05:08:53.314781888Z (EngineCore pid=169) INFO 10-15 05:08:53 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T05:08:54.079567248Z (EngineCore pid=169) INFO 10-15 05:08:54 [model_runner.py:396] Loading model from scratch... +2026-10-15T05:08:54.655291043Z (EngineCore pid=169) INFO 10-15 05:08:54 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T05:08:54.655331658Z (EngineCore pid=169) INFO 10-15 05:08:54 [flash_attn.py:1116] Using FlashAttention version 3 +2026-10-15T05:08:54.768638229Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T05:08:54.768672205Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T05:08:55.068635806Z (EngineCore pid=169) INFO 10-15 05:08:55 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. +2026-10-15T05:08:55.068664846Z (EngineCore pid=169) INFO 10-15 05:08:55 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T05:08:55.069420485Z (EngineCore pid=169) +Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T05:06:33.868612492Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T05:06:33.868619724Z +2026-10-15T05:06:34.074918680Z Loading base model Qwen/Qwen3-8B to cuda:0... +2026-10-15T05:06:34.296341094Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T05:06:35.101730406Z [prefetch] reading 5 file(s), 15.26 GiB, from nfs into the page cache with 8 threads +2026-10-15T05:06:44.477515651Z [prefetch] done: 15.26 GiB in 9.4 s (1666 MiB/s) +2026-10-15T05:06:46.952210680Z +Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json new file mode 100644 index 00000000..b2c3f8d6 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "generateName": "open-rl-api-server-6d86b498f5-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "6d86b498f5", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + } + ], + "resourceVersion": "1792039570790271003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2ns79", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:59Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:45:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.11", + "podIPs": [ + { + "ip": "10.32.3.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:45:46Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:58Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-sampler", + "uid": "3920e58e-f415-4c96-8485-77a8cd538872" + } + ], + "resourceVersion": "1792040908564591023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-sampler-3920e58e" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "44791173-6b7c-4d22-9a8b-46ef83190384" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-vnbl2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-vnbl2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:08:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:06:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:08:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:08:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:06:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "40Gi" + }, + "containerID": "containerd://3e4a6241afecb5338cb06427e1604c19bcecb3758c35b1a359aa2774bbab9c1b", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:08:28Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-vnbl2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.1.9", + "podIPs": [ + { + "ip": "10.32.1.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:06:58Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:59Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-trainer", + "uid": "125fac70-8359-4f66-bcb5-20df4c1e1764" + } + ], + "resourceVersion": "1792040789855759006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-trainer-125fac70" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "371ea4b9-ae90-4e15-a7b1-b34802637e23" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5vhhk", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-5vhhk", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:06:29Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:04:59Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:06:29Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:06:29Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:04:59Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "36Gi" + }, + "containerID": "containerd://e6e0c04018d8bd44ea79dcdecc69553e9707fd65313f4196c6ac54d3260924ce", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:06:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5vhhk", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.4.10", + "podIPs": [ + { + "ip": "10.32.4.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:04:59Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log new file mode 100644 index 00000000..fef5c86f --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log @@ -0,0 +1,575 @@ +2026-10-15T05:04:55.021+00:00 2026-10-15T05:04:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:04:56.026+00:00 2026-10-15T05:04:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:04:57.031+00:00 2026-10-15T05:04:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:04:58.037+00:00 2026-10-15T05:04:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:04:59.042+00:00 2026-10-15T05:04:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:00.047+00:00 2026-10-15T05:05:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:01.055+00:00 2026-10-15T05:05:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:02.059+00:00 2026-10-15T05:05:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:03.066+00:00 2026-10-15T05:05:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:04.069+00:00 2026-10-15T05:05:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:05.074+00:00 2026-10-15T05:05:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:06.079+00:00 2026-10-15T05:05:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:07.085+00:00 2026-10-15T05:05:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:08.092+00:00 2026-10-15T05:05:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:09.097+00:00 2026-10-15T05:05:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:10.100+00:00 2026-10-15T05:05:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:11.106+00:00 2026-10-15T05:05:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:12.112+00:00 2026-10-15T05:05:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:13.118+00:00 2026-10-15T05:05:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:14.123+00:00 2026-10-15T05:05:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:15.129+00:00 2026-10-15T05:05:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:16.134+00:00 2026-10-15T05:05:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:17.143+00:00 2026-10-15T05:05:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:18.143+00:00 2026-10-15T05:05:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:19.148+00:00 2026-10-15T05:05:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:20.152+00:00 2026-10-15T05:05:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:21.158+00:00 2026-10-15T05:05:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:22.163+00:00 2026-10-15T05:05:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:23.168+00:00 2026-10-15T05:05:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:24.179+00:00 2026-10-15T05:05:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:25.179+00:00 2026-10-15T05:05:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:26.184+00:00 2026-10-15T05:05:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:27.195+00:00 2026-10-15T05:05:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:28.194+00:00 2026-10-15T05:05:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:29.198+00:00 2026-10-15T05:05:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:30.204+00:00 2026-10-15T05:05:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:31.210+00:00 2026-10-15T05:05:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:32.214+00:00 2026-10-15T05:05:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:33.220+00:00 2026-10-15T05:05:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:34.225+00:00 2026-10-15T05:05:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:35.231+00:00 2026-10-15T05:05:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:36.236+00:00 2026-10-15T05:05:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:37.242+00:00 2026-10-15T05:05:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:38.247+00:00 2026-10-15T05:05:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:39.252+00:00 2026-10-15T05:05:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:40.257+00:00 2026-10-15T05:05:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:41.262+00:00 2026-10-15T05:05:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:42.267+00:00 2026-10-15T05:05:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:43.272+00:00 2026-10-15T05:05:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:44.278+00:00 2026-10-15T05:05:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:45.282+00:00 2026-10-15T05:05:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:46.287+00:00 2026-10-15T05:05:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:47.292+00:00 2026-10-15T05:05:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:48.365+00:00 2026-10-15T05:05:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:49.304+00:00 2026-10-15T05:05:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:50.306+00:00 2026-10-15T05:05:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:51.313+00:00 2026-10-15T05:05:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:52.317+00:00 2026-10-15T05:05:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:53.322+00:00 2026-10-15T05:05:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:54.328+00:00 2026-10-15T05:05:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:55.344+00:00 2026-10-15T05:05:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:56.338+00:00 2026-10-15T05:05:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:57.347+00:00 2026-10-15T05:05:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:58.348+00:00 2026-10-15T05:05:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:05:59.353+00:00 2026-10-15T05:05:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:00.358+00:00 2026-10-15T05:06:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:01.365+00:00 2026-10-15T05:06:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:02.370+00:00 2026-10-15T05:06:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:03.375+00:00 2026-10-15T05:06:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:04.381+00:00 2026-10-15T05:06:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:05.388+00:00 2026-10-15T05:06:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:06.400+00:00 2026-10-15T05:06:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:07.395+00:00 2026-10-15T05:06:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:08.401+00:00 2026-10-15T05:06:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:09.406+00:00 2026-10-15T05:06:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:10.411+00:00 2026-10-15T05:06:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:11.416+00:00 2026-10-15T05:06:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:12.422+00:00 2026-10-15T05:06:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:13.427+00:00 2026-10-15T05:06:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:14.433+00:00 2026-10-15T05:06:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:15.437+00:00 2026-10-15T05:06:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:16.457+00:00 2026-10-15T05:06:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:17.453+00:00 2026-10-15T05:06:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:18.453+00:00 2026-10-15T05:06:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:19.458+00:00 2026-10-15T05:06:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:20.463+00:00 2026-10-15T05:06:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:21.468+00:00 2026-10-15T05:06:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:22.472+00:00 2026-10-15T05:06:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:23.478+00:00 2026-10-15T05:06:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:24.483+00:00 2026-10-15T05:06:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:25.488+00:00 2026-10-15T05:06:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:26.493+00:00 2026-10-15T05:06:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:27.498+00:00 2026-10-15T05:06:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:28.517+00:00 2026-10-15T05:06:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:29.508+00:00 2026-10-15T05:06:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:30.513+00:00 2026-10-15T05:06:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:31.527+00:00 2026-10-15T05:06:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:32.523+00:00 2026-10-15T05:06:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:33.528+00:00 2026-10-15T05:06:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:34.533+00:00 2026-10-15T05:06:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:35.607+00:00 2026-10-15T05:06:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:36.545+00:00 2026-10-15T05:06:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:37.548+00:00 2026-10-15T05:06:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:38.553+00:00 2026-10-15T05:06:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:39.558+00:00 2026-10-15T05:06:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:40.563+00:00 2026-10-15T05:06:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:41.569+00:00 2026-10-15T05:06:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:42.574+00:00 2026-10-15T05:06:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:43.579+00:00 2026-10-15T05:06:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:44.591+00:00 2026-10-15T05:06:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:45.589+00:00 2026-10-15T05:06:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:46.600+00:00 2026-10-15T05:06:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:47.598+00:00 2026-10-15T05:06:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:47.598+00:00 2026-10-15T05:06:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:48.603+00:00 2026-10-15T05:06:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:48.603+00:00 2026-10-15T05:06:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:49.608+00:00 2026-10-15T05:06:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:49.608+00:00 2026-10-15T05:06:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:50.613+00:00 2026-10-15T05:06:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:50.613+00:00 2026-10-15T05:06:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:51.618+00:00 2026-10-15T05:06:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:51.618+00:00 2026-10-15T05:06:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:52.623+00:00 2026-10-15T05:06:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:52.623+00:00 2026-10-15T05:06:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:53.635+00:00 2026-10-15T05:06:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:53.635+00:00 2026-10-15T05:06:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:54.632+00:00 2026-10-15T05:06:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:54.632+00:00 2026-10-15T05:06:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:55.637+00:00 2026-10-15T05:06:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:55.637+00:00 2026-10-15T05:06:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:56.775+00:00 2026-10-15T05:06:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:56.775+00:00 2026-10-15T05:06:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:57.648+00:00 2026-10-15T05:06:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:57.648+00:00 2026-10-15T05:06:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:58.652+00:00 2026-10-15T05:06:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:58.652+00:00 2026-10-15T05:06:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:06:59.709+00:00 2026-10-15T05:06:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:06:59.709+00:00 2026-10-15T05:06:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:00.664+00:00 2026-10-15T05:07:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:00.664+00:00 2026-10-15T05:07:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:01.667+00:00 2026-10-15T05:07:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:01.667+00:00 2026-10-15T05:07:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:02.672+00:00 2026-10-15T05:07:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:02.672+00:00 2026-10-15T05:07:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:03.749+00:00 2026-10-15T05:07:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:03.749+00:00 2026-10-15T05:07:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:04.681+00:00 2026-10-15T05:07:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:04.681+00:00 2026-10-15T05:07:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:05.687+00:00 2026-10-15T05:07:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:05.687+00:00 2026-10-15T05:07:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:06.692+00:00 2026-10-15T05:07:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:06.692+00:00 2026-10-15T05:07:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:07.700+00:00 2026-10-15T05:07:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:07.700+00:00 2026-10-15T05:07:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:08.701+00:00 2026-10-15T05:07:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:08.701+00:00 2026-10-15T05:07:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:09.705+00:00 2026-10-15T05:07:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:09.705+00:00 2026-10-15T05:07:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:10.714+00:00 2026-10-15T05:07:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:10.714+00:00 2026-10-15T05:07:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:11.716+00:00 2026-10-15T05:07:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:11.716+00:00 2026-10-15T05:07:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:12.720+00:00 2026-10-15T05:07:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:12.720+00:00 2026-10-15T05:07:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:13.725+00:00 2026-10-15T05:07:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:13.725+00:00 2026-10-15T05:07:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:14.730+00:00 2026-10-15T05:07:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:14.731+00:00 2026-10-15T05:07:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:15.735+00:00 2026-10-15T05:07:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:15.735+00:00 2026-10-15T05:07:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:16.740+00:00 2026-10-15T05:07:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:16.740+00:00 2026-10-15T05:07:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:17.746+00:00 2026-10-15T05:07:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:17.746+00:00 2026-10-15T05:07:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:18.750+00:00 2026-10-15T05:07:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:18.750+00:00 2026-10-15T05:07:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:19.756+00:00 2026-10-15T05:07:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:19.757+00:00 2026-10-15T05:07:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:20.760+00:00 2026-10-15T05:07:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:20.761+00:00 2026-10-15T05:07:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:21.765+00:00 2026-10-15T05:07:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:21.765+00:00 2026-10-15T05:07:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:22.770+00:00 2026-10-15T05:07:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:22.770+00:00 2026-10-15T05:07:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:23.775+00:00 2026-10-15T05:07:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:23.775+00:00 2026-10-15T05:07:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:24.781+00:00 2026-10-15T05:07:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:24.781+00:00 2026-10-15T05:07:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:25.786+00:00 2026-10-15T05:07:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:25.786+00:00 2026-10-15T05:07:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:26.790+00:00 2026-10-15T05:07:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:26.790+00:00 2026-10-15T05:07:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:27.902+00:00 2026-10-15T05:07:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:27.902+00:00 2026-10-15T05:07:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:28.800+00:00 2026-10-15T05:07:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:28.800+00:00 2026-10-15T05:07:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:29.805+00:00 2026-10-15T05:07:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:29.805+00:00 2026-10-15T05:07:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:30.812+00:00 2026-10-15T05:07:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:30.812+00:00 2026-10-15T05:07:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:31.816+00:00 2026-10-15T05:07:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:31.816+00:00 2026-10-15T05:07:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:32.821+00:00 2026-10-15T05:07:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:32.821+00:00 2026-10-15T05:07:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:33.826+00:00 2026-10-15T05:07:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:33.826+00:00 2026-10-15T05:07:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:34.832+00:00 2026-10-15T05:07:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:34.833+00:00 2026-10-15T05:07:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:35.837+00:00 2026-10-15T05:07:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:35.837+00:00 2026-10-15T05:07:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:36.843+00:00 2026-10-15T05:07:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:36.843+00:00 2026-10-15T05:07:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:37.847+00:00 2026-10-15T05:07:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:37.847+00:00 2026-10-15T05:07:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:38.852+00:00 2026-10-15T05:07:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:38.853+00:00 2026-10-15T05:07:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:39.859+00:00 2026-10-15T05:07:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:39.859+00:00 2026-10-15T05:07:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:40.864+00:00 2026-10-15T05:07:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:40.864+00:00 2026-10-15T05:07:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:41.869+00:00 2026-10-15T05:07:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:41.869+00:00 2026-10-15T05:07:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:42.875+00:00 2026-10-15T05:07:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:42.875+00:00 2026-10-15T05:07:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:43.880+00:00 2026-10-15T05:07:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:43.880+00:00 2026-10-15T05:07:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:44.890+00:00 2026-10-15T05:07:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:44.890+00:00 2026-10-15T05:07:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:45.890+00:00 2026-10-15T05:07:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:45.890+00:00 2026-10-15T05:07:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:46.904+00:00 2026-10-15T05:07:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:46.905+00:00 2026-10-15T05:07:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:47.899+00:00 2026-10-15T05:07:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:47.899+00:00 2026-10-15T05:07:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:48.906+00:00 2026-10-15T05:07:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:48.906+00:00 2026-10-15T05:07:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:49.910+00:00 2026-10-15T05:07:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:49.910+00:00 2026-10-15T05:07:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:50.916+00:00 2026-10-15T05:07:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:50.916+00:00 2026-10-15T05:07:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:51.921+00:00 2026-10-15T05:07:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:51.922+00:00 2026-10-15T05:07:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:52.928+00:00 2026-10-15T05:07:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:52.929+00:00 2026-10-15T05:07:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:53.931+00:00 2026-10-15T05:07:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:53.931+00:00 2026-10-15T05:07:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:54.935+00:00 2026-10-15T05:07:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:54.935+00:00 2026-10-15T05:07:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:55.951+00:00 2026-10-15T05:07:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:55.951+00:00 2026-10-15T05:07:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:56.945+00:00 2026-10-15T05:07:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:56.945+00:00 2026-10-15T05:07:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:57.974+00:00 2026-10-15T05:07:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:57.974+00:00 2026-10-15T05:07:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:58.955+00:00 2026-10-15T05:07:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:58.955+00:00 2026-10-15T05:07:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:07:59.960+00:00 2026-10-15T05:07:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:07:59.960+00:00 2026-10-15T05:07:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:00.965+00:00 2026-10-15T05:08:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:00.965+00:00 2026-10-15T05:08:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:01.970+00:00 2026-10-15T05:08:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:01.970+00:00 2026-10-15T05:08:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:02.978+00:00 2026-10-15T05:08:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:02.978+00:00 2026-10-15T05:08:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:03.984+00:00 2026-10-15T05:08:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:03.984+00:00 2026-10-15T05:08:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:04.990+00:00 2026-10-15T05:08:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:04.990+00:00 2026-10-15T05:08:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:06.000+00:00 2026-10-15T05:08:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:06.000+00:00 2026-10-15T05:08:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:07.000+00:00 2026-10-15T05:08:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:07.000+00:00 2026-10-15T05:08:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:08.008+00:00 2026-10-15T05:08:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:08.008+00:00 2026-10-15T05:08:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:09.010+00:00 2026-10-15T05:08:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:09.010+00:00 2026-10-15T05:08:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:10.016+00:00 2026-10-15T05:08:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:10.016+00:00 2026-10-15T05:08:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:11.019+00:00 2026-10-15T05:08:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:11.020+00:00 2026-10-15T05:08:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:12.025+00:00 2026-10-15T05:08:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:12.026+00:00 2026-10-15T05:08:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:13.030+00:00 2026-10-15T05:08:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:13.030+00:00 2026-10-15T05:08:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:14.048+00:00 2026-10-15T05:08:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:14.048+00:00 2026-10-15T05:08:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:15.040+00:00 2026-10-15T05:08:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:15.040+00:00 2026-10-15T05:08:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:16.049+00:00 2026-10-15T05:08:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:16.050+00:00 2026-10-15T05:08:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:17.053+00:00 2026-10-15T05:08:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:17.053+00:00 2026-10-15T05:08:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:18.058+00:00 2026-10-15T05:08:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:18.058+00:00 2026-10-15T05:08:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:19.062+00:00 2026-10-15T05:08:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:19.062+00:00 2026-10-15T05:08:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:20.066+00:00 2026-10-15T05:08:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:20.066+00:00 2026-10-15T05:08:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:21.076+00:00 2026-10-15T05:08:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:21.076+00:00 2026-10-15T05:08:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:22.078+00:00 2026-10-15T05:08:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:22.078+00:00 2026-10-15T05:08:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:23.084+00:00 2026-10-15T05:08:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:23.084+00:00 2026-10-15T05:08:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:24.092+00:00 2026-10-15T05:08:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:24.092+00:00 2026-10-15T05:08:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:25.094+00:00 2026-10-15T05:08:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:25.094+00:00 2026-10-15T05:08:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:26.100+00:00 2026-10-15T05:08:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:26.100+00:00 2026-10-15T05:08:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:27.103+00:00 2026-10-15T05:08:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:27.104+00:00 2026-10-15T05:08:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:28.108+00:00 2026-10-15T05:08:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:28.108+00:00 2026-10-15T05:08:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:29.113+00:00 2026-10-15T05:08:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:29.113+00:00 2026-10-15T05:08:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:30.119+00:00 2026-10-15T05:08:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:30.119+00:00 2026-10-15T05:08:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:31.124+00:00 2026-10-15T05:08:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:31.124+00:00 2026-10-15T05:08:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:32.129+00:00 2026-10-15T05:08:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:32.129+00:00 2026-10-15T05:08:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:33.133+00:00 2026-10-15T05:08:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:33.133+00:00 2026-10-15T05:08:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:34.139+00:00 2026-10-15T05:08:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:34.139+00:00 2026-10-15T05:08:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:35.143+00:00 2026-10-15T05:08:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:35.143+00:00 2026-10-15T05:08:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:36.321+00:00 2026-10-15T05:08:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:36.321+00:00 2026-10-15T05:08:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:37.153+00:00 2026-10-15T05:08:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:37.153+00:00 2026-10-15T05:08:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:38.158+00:00 2026-10-15T05:08:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:38.158+00:00 2026-10-15T05:08:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:39.178+00:00 2026-10-15T05:08:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:39.178+00:00 2026-10-15T05:08:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:40.168+00:00 2026-10-15T05:08:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:40.168+00:00 2026-10-15T05:08:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:41.172+00:00 2026-10-15T05:08:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:41.172+00:00 2026-10-15T05:08:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:42.179+00:00 2026-10-15T05:08:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:42.179+00:00 2026-10-15T05:08:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:43.183+00:00 2026-10-15T05:08:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:43.183+00:00 2026-10-15T05:08:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:44.188+00:00 2026-10-15T05:08:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:44.188+00:00 2026-10-15T05:08:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:45.192+00:00 2026-10-15T05:08:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:45.193+00:00 2026-10-15T05:08:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:46.198+00:00 2026-10-15T05:08:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:46.198+00:00 2026-10-15T05:08:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:47.203+00:00 2026-10-15T05:08:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:47.203+00:00 2026-10-15T05:08:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:48.207+00:00 2026-10-15T05:08:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:48.207+00:00 2026-10-15T05:08:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:49.212+00:00 2026-10-15T05:08:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:49.212+00:00 2026-10-15T05:08:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:50.217+00:00 2026-10-15T05:08:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:50.217+00:00 2026-10-15T05:08:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:51.256+00:00 2026-10-15T05:08:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:51.257+00:00 2026-10-15T05:08:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:52.226+00:00 2026-10-15T05:08:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:52.226+00:00 2026-10-15T05:08:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:53.231+00:00 2026-10-15T05:08:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:53.231+00:00 2026-10-15T05:08:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:54.236+00:00 2026-10-15T05:08:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:54.236+00:00 2026-10-15T05:08:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:55.241+00:00 2026-10-15T05:08:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:55.241+00:00 2026-10-15T05:08:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:56.247+00:00 2026-10-15T05:08:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:56.247+00:00 2026-10-15T05:08:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:57.269+00:00 2026-10-15T05:08:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:57.269+00:00 2026-10-15T05:08:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:58.258+00:00 2026-10-15T05:08:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:58.258+00:00 2026-10-15T05:08:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:08:59.262+00:00 2026-10-15T05:08:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:08:59.262+00:00 2026-10-15T05:08:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:00.271+00:00 2026-10-15T05:09:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:00.271+00:00 2026-10-15T05:09:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:01.272+00:00 2026-10-15T05:09:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:01.272+00:00 2026-10-15T05:09:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:02.283+00:00 2026-10-15T05:09:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:02.283+00:00 2026-10-15T05:09:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:03.282+00:00 2026-10-15T05:09:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:03.282+00:00 2026-10-15T05:09:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:04.291+00:00 2026-10-15T05:09:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:04.291+00:00 2026-10-15T05:09:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:05.293+00:00 2026-10-15T05:09:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:05.294+00:00 2026-10-15T05:09:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:06.299+00:00 2026-10-15T05:09:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:06.299+00:00 2026-10-15T05:09:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:07.304+00:00 2026-10-15T05:09:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:07.304+00:00 2026-10-15T05:09:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:08.307+00:00 2026-10-15T05:09:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:08.308+00:00 2026-10-15T05:09:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:09.312+00:00 2026-10-15T05:09:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:09.312+00:00 2026-10-15T05:09:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:10.318+00:00 2026-10-15T05:09:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:10.318+00:00 2026-10-15T05:09:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:11.321+00:00 2026-10-15T05:09:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:11.321+00:00 2026-10-15T05:09:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:12.327+00:00 2026-10-15T05:09:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:12.328+00:00 2026-10-15T05:09:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:13.332+00:00 2026-10-15T05:09:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:13.333+00:00 2026-10-15T05:09:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:14.392+00:00 2026-10-15T05:09:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:14.392+00:00 2026-10-15T05:09:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:15.342+00:00 2026-10-15T05:09:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:15.342+00:00 2026-10-15T05:09:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:16.348+00:00 2026-10-15T05:09:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:16.348+00:00 2026-10-15T05:09:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:17.352+00:00 2026-10-15T05:09:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:17.353+00:00 2026-10-15T05:09:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:18.356+00:00 2026-10-15T05:09:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:18.356+00:00 2026-10-15T05:09:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:19.362+00:00 2026-10-15T05:09:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:19.362+00:00 2026-10-15T05:09:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:20.370+00:00 2026-10-15T05:09:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:20.370+00:00 2026-10-15T05:09:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:21.372+00:00 2026-10-15T05:09:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:21.372+00:00 2026-10-15T05:09:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:22.392+00:00 2026-10-15T05:09:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:22.392+00:00 2026-10-15T05:09:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:23.381+00:00 2026-10-15T05:09:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:23.381+00:00 2026-10-15T05:09:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:24.386+00:00 2026-10-15T05:09:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:24.386+00:00 2026-10-15T05:09:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:25.391+00:00 2026-10-15T05:09:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:25.391+00:00 2026-10-15T05:09:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:26.397+00:00 2026-10-15T05:09:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:26.397+00:00 2026-10-15T05:09:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:27.404+00:00 2026-10-15T05:09:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:27.404+00:00 2026-10-15T05:09:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:28.411+00:00 2026-10-15T05:09:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:28.412+00:00 2026-10-15T05:09:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:29.416+00:00 2026-10-15T05:09:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:29.416+00:00 2026-10-15T05:09:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:30.417+00:00 2026-10-15T05:09:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:30.417+00:00 2026-10-15T05:09:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:31.515+00:00 2026-10-15T05:09:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:31.515+00:00 2026-10-15T05:09:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:32.426+00:00 2026-10-15T05:09:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:32.426+00:00 2026-10-15T05:09:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:33.431+00:00 2026-10-15T05:09:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:33.431+00:00 2026-10-15T05:09:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:34.436+00:00 2026-10-15T05:09:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:34.437+00:00 2026-10-15T05:09:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:35.443+00:00 2026-10-15T05:09:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:35.443+00:00 2026-10-15T05:09:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:36.447+00:00 2026-10-15T05:09:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:36.447+00:00 2026-10-15T05:09:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:37.451+00:00 2026-10-15T05:09:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:37.451+00:00 2026-10-15T05:09:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:38.460+00:00 2026-10-15T05:09:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:38.460+00:00 2026-10-15T05:09:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:39.462+00:00 2026-10-15T05:09:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:39.462+00:00 2026-10-15T05:09:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:40.467+00:00 2026-10-15T05:09:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:40.467+00:00 2026-10-15T05:09:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:41.472+00:00 2026-10-15T05:09:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:41.472+00:00 2026-10-15T05:09:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:42.477+00:00 2026-10-15T05:09:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:42.477+00:00 2026-10-15T05:09:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:43.483+00:00 2026-10-15T05:09:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:43.483+00:00 2026-10-15T05:09:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:44.489+00:00 2026-10-15T05:09:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:44.489+00:00 2026-10-15T05:09:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:45.496+00:00 2026-10-15T05:09:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:45.496+00:00 2026-10-15T05:09:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:46.499+00:00 2026-10-15T05:09:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:46.499+00:00 2026-10-15T05:09:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:47.504+00:00 2026-10-15T05:09:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:47.504+00:00 2026-10-15T05:09:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:48.509+00:00 2026-10-15T05:09:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:48.509+00:00 2026-10-15T05:09:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:49.515+00:00 2026-10-15T05:09:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:49.515+00:00 2026-10-15T05:09:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:50.520+00:00 2026-10-15T05:09:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:50.521+00:00 2026-10-15T05:09:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:51.525+00:00 2026-10-15T05:09:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:51.526+00:00 2026-10-15T05:09:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:52.531+00:00 2026-10-15T05:09:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:52.531+00:00 2026-10-15T05:09:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:53.542+00:00 2026-10-15T05:09:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:53.542+00:00 2026-10-15T05:09:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:54.542+00:00 2026-10-15T05:09:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:54.542+00:00 2026-10-15T05:09:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:55.546+00:00 2026-10-15T05:09:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:55.546+00:00 2026-10-15T05:09:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:56.552+00:00 2026-10-15T05:09:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:56.552+00:00 2026-10-15T05:09:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:57.558+00:00 2026-10-15T05:09:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:57.559+00:00 2026-10-15T05:09:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:58.573+00:00 2026-10-15T05:09:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:58.573+00:00 2026-10-15T05:09:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:09:59.566+00:00 2026-10-15T05:09:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:09:59.566+00:00 2026-10-15T05:09:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:00.573+00:00 2026-10-15T05:10:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:00.573+00:00 2026-10-15T05:10:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:01.579+00:00 2026-10-15T05:10:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:01.579+00:00 2026-10-15T05:10:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:02.586+00:00 2026-10-15T05:10:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:02.586+00:00 2026-10-15T05:10:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:03.595+00:00 2026-10-15T05:10:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:03.596+00:00 2026-10-15T05:10:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:04.597+00:00 2026-10-15T05:10:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:04.597+00:00 2026-10-15T05:10:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:05.603+00:00 2026-10-15T05:10:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:05.603+00:00 2026-10-15T05:10:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:06.609+00:00 2026-10-15T05:10:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:06.609+00:00 2026-10-15T05:10:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:07.618+00:00 2026-10-15T05:10:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:07.618+00:00 2026-10-15T05:10:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:08.623+00:00 2026-10-15T05:10:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:08.623+00:00 2026-10-15T05:10:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:09.629+00:00 2026-10-15T05:10:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:09.629+00:00 2026-10-15T05:10:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:10.636+00:00 2026-10-15T05:10:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:10.636+00:00 2026-10-15T05:10:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:11.642+00:00 2026-10-15T05:10:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:11.642+00:00 2026-10-15T05:10:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:12.647+00:00 2026-10-15T05:10:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:12.647+00:00 2026-10-15T05:10:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:13.651+00:00 2026-10-15T05:10:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:13.651+00:00 2026-10-15T05:10:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:14.657+00:00 2026-10-15T05:10:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:14.658+00:00 2026-10-15T05:10:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:15.803+00:00 2026-10-15T05:10:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:15.803+00:00 2026-10-15T05:10:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:16.680+00:00 2026-10-15T05:10:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:16.680+00:00 2026-10-15T05:10:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:17.672+00:00 2026-10-15T05:10:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:17.672+00:00 2026-10-15T05:10:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:18.710+00:00 2026-10-15T05:10:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:18.711+00:00 2026-10-15T05:10:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:19.683+00:00 2026-10-15T05:10:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:19.683+00:00 2026-10-15T05:10:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:20.688+00:00 2026-10-15T05:10:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:20.688+00:00 2026-10-15T05:10:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:21.693+00:00 2026-10-15T05:10:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:21.694+00:00 2026-10-15T05:10:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:22.699+00:00 2026-10-15T05:10:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:22.699+00:00 2026-10-15T05:10:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:23.703+00:00 2026-10-15T05:10:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:23.703+00:00 2026-10-15T05:10:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:24.708+00:00 2026-10-15T05:10:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:24.708+00:00 2026-10-15T05:10:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:25.714+00:00 2026-10-15T05:10:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:25.715+00:00 2026-10-15T05:10:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:26.720+00:00 2026-10-15T05:10:26Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:26.720+00:00 2026-10-15T05:10:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:26.720+00:00 2026-10-15T05:10:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:27.726+00:00 2026-10-15T05:10:27Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:27.726+00:00 2026-10-15T05:10:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:27.726+00:00 2026-10-15T05:10:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:28.730+00:00 2026-10-15T05:10:28Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:28.731+00:00 2026-10-15T05:10:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:28.731+00:00 2026-10-15T05:10:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:29.741+00:00 2026-10-15T05:10:29Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:29.741+00:00 2026-10-15T05:10:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:29.741+00:00 2026-10-15T05:10:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:30.740+00:00 2026-10-15T05:10:30Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:30.740+00:00 2026-10-15T05:10:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:30.740+00:00 2026-10-15T05:10:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:31.746+00:00 2026-10-15T05:10:31Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:31.746+00:00 2026-10-15T05:10:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:31.746+00:00 2026-10-15T05:10:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:32.755+00:00 2026-10-15T05:10:32Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:32.755+00:00 2026-10-15T05:10:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:32.755+00:00 2026-10-15T05:10:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:33.755+00:00 2026-10-15T05:10:33Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:33.756+00:00 2026-10-15T05:10:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:33.756+00:00 2026-10-15T05:10:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:10:34.761+00:00 2026-10-15T05:10:34Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:10:34.761+00:00 2026-10-15T05:10:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:10:34.761+00:00 2026-10-15T05:10:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log new file mode 100644 index 00000000..3f20cd2a --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T05:04:59.071+00:00 2026-10-15T05:04:59.090098465Z {"level":"info","ts":"2026-10-15T05:04:59Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"065c25b1-0fba-4abe-8200-01346b4d1314","claim":"claim-lora-qwen-qwen3-8b-0-trainer-125fac70","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} +2026-10-15T05:06:58.207+00:00 2026-10-15T05:06:58.027203485Z {"level":"info","ts":"2026-10-15T05:06:58Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"ccc8816d-971a-4ae8-bad0-64691bfb74c6","claim":"claim-lora-qwen-qwen3-8b-0-sampler-3920e58e","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json new file mode 100644 index 00000000..c2297ac3 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json @@ -0,0 +1,275 @@ +{ + "case": "after-C1-8b-h100-cold", + "collected": "2026-10-15T05:10:35.141+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 05:04:58.138000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen3-8B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 05:04:58.139000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 05:06:49.594000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "048101ca-2c60-4c63-afa2-2ae75e83f1c4" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 05:06:52.784000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 05:06:55.595000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 05:06:57.617000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 05:06:57.618000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 05:06:59.533000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 05:06:59.534000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 05:10:27.431000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4\n\nQuestion: What is " + }, + "client.end": { + "ts": "2026-10-15 05:10:27.434000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen3-8b-0-sampler", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "pod": "orw-lora-qwen-qwen3-8b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 05:06:57+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:06:56.988000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 05:06:56.988000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 05:06:56.988000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 05:08:28+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 05:06:58+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 05:06:58+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 05:08:28+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 05:08:28+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 05:06:58+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 05:08:28+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 05:06:58+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "sampler.event_Pulling": { + "ts": "2026-10-15 05:07:01+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 05:08:28+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.45", + "kubelet_pull_s": 86.459, + "pulled_from_cache": false + }, + "sampler.event_Created": { + "ts": "2026-10-15 05:08:28+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 05:08:28+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 05:08:35.508944+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 05:08:52.969476+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:08:52 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 05:08:54.079567+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:08:54 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 05:08:55.068664+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:08:55 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 05:09:08.096254+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:09:08 [default_loader.py:430] Loading weights took 13.03 seconds", + "self_reported_s": 13.03 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 05:09:08.762926+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:09:08 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.956827 seconds", + "self_reported_s": 14.956827 + }, + "sampler.compile_done": { + "ts": "2026-10-15 05:09:45.574813+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:09:45 [monitor.py:53] torch.compile took 35.82 s in total", + "self_reported_s": 35.82 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 05:10:06.072864+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:10:06 [model_runner.py:1066] Graph capturing finished in 17 secs, took 0.78 GiB", + "self_reported_s": 17.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 05:10:06.701571+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:10:06 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 05:10:25.431150+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:10:25 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.66 s", + "self_reported_s": 76.66 + }, + "sampler.ready_key": { + "ts": "2026-10-15 05:10:26+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + }, + "trainer": { + "workload": "lora-qwen-qwen3-8b-0-trainer", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "pod": "orw-lora-qwen-qwen3-8b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 05:04:58+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:04:58.742000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 05:04:58.742000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 05:05:00.362000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 05:06:30+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 05:04:59+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 05:04:59+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 05:06:29+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 05:06:29+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 05:04:59+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 05:06:29+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 05:04:59+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "trainer.event_Pulling": { + "ts": "2026-10-15 05:05:02+00:00", + "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 05:06:29+00:00", + "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.47", + "kubelet_pull_s": 86.476, + "pulled_from_cache": false + }, + "trainer.event_Created": { + "ts": "2026-10-15 05:06:29+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 05:06:29+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 05:06:33.850821+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 05:06:34.074918+00:00", + "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 05:06:47.108478+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 05:06:47.110263+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 05:06:47+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.md b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.md new file mode 100644 index 00000000..eaae3c8d --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.md @@ -0,0 +1,111 @@ +# Cold start timeline: after-C1-8b-h100-cold + +t=0 is the client's `create_model` call at 2026-10-15T05:04:58.139+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen3-8b-0-trainer` on `gke-open-rl-dra-h100-trainer-702095b4-1g3a` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.workload_created | 05:04:58.000 | -0.1 | | Workload creationTimestamp (apiserver) | +| trainer.workload_first_seen_by_poller | 05:04:58.742 | 0.6 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 05:04:58.742 | 0.6 | | workloads.tsv poller: status.podName set | +| trainer.pod_created | 05:04:59.000 | 0.9 | | pod creationTimestamp (apiserver) | +| trainer.pod_Initialized | 05:04:59.000 | 0.9 | | pod condition Initialized | +| trainer.pod_PodScheduled | 05:04:59.000 | 0.9 | | pod condition PodScheduled | +| trainer.event_Scheduled | 05:04:59.000 | 0.9 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a | +| trainer.workload_reports_node | 05:05:00.362 | 2.2 | | workloads.tsv poller: status.nodeName set | +| trainer.event_Pulling | 05:05:02.000 | 3.9 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" | +| trainer.pod_Ready | 05:06:29.000 | 90.9 | | pod condition Ready | +| trainer.pod_ContainersReady | 05:06:29.000 | 90.9 | | pod condition ContainersReady | +| trainer.container_started | 05:06:29.000 | 90.9 | | containerStatuses.state.running.startedAt | +| trainer.event_Pulled | 05:06:29.000 | 90.9 | 86.476 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" in 1m26.476s (1m26.47 | +| trainer.event_Created | 05:06:29.000 | 90.9 | | event: Container created | +| trainer.event_Started | 05:06:29.000 | 90.9 | | event: Container started | +| trainer.condition_Placed_True | 05:06:30.000 | 91.9 | | Workload condition Placed | +| trainer.process_first_log | 05:06:33.850 | 95.7 | | container log, first line | +| trainer.load_start | 05:06:34.074 | 95.9 | | container log: Loading base model Qwen/Qwen3-8B to cuda:0... | +| trainer.ready_key | 05:06:47.000 | 108.9 | | redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B | +| trainer.load_done | 05:06:47.108 | 109.0 | | container log: Successfully loaded. | +| trainer.processor_started | 05:06:47.110 | 109.0 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1). | + +## sampler: `orw-lora-qwen-qwen3-8b-0-sampler` on `gke-open-rl-dra-h100-sampler-a721f127-0853` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_first_seen_by_poller | 05:06:56.988 | 118.8 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_reports_pod | 05:06:56.988 | 118.8 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 05:06:56.988 | 118.8 | | workloads.tsv poller: status.nodeName set | +| sampler.workload_created | 05:06:57.000 | 118.9 | | Workload creationTimestamp (apiserver) | +| sampler.pod_created | 05:06:58.000 | 119.9 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 05:06:58.000 | 119.9 | | pod condition Initialized | +| sampler.pod_PodScheduled | 05:06:58.000 | 119.9 | | pod condition PodScheduled | +| sampler.event_Scheduled | 05:06:58.000 | 119.9 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853 | +| sampler.event_Pulling | 05:07:01.000 | 122.9 | | event: Pulling image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" | +| sampler.condition_Placed_True | 05:08:28.000 | 209.9 | | Workload condition Placed | +| sampler.pod_Ready | 05:08:28.000 | 209.9 | | pod condition Ready | +| sampler.pod_ContainersReady | 05:08:28.000 | 209.9 | | pod condition ContainersReady | +| sampler.container_started | 05:08:28.000 | 209.9 | | containerStatuses.state.running.startedAt | +| sampler.event_Pulled | 05:08:28.000 | 209.9 | 86.459 | event: Successfully pulled image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" in 1m26.459s (1m26.45 | +| sampler.event_Created | 05:08:28.000 | 209.9 | | event: Container created | +| sampler.event_Started | 05:08:28.000 | 209.9 | | event: Container started | +| sampler.process_first_log | 05:08:35.508 | 217.4 | | container log, first line | +| sampler.engine_init_start | 05:08:52.969 | 234.8 | | container log: (EngineCore pid=169) INFO 10-15 05:08:52 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 05:08:54.079 | 235.9 | | container log: (EngineCore pid=169) INFO 10-15 05:08:54 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 05:08:55.068 | 236.9 | | container log: (EngineCore pid=169) INFO 10-15 05:08:55 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 05:09:08.096 | 250.0 | 13.03 | container log: (EngineCore pid=169) INFO 10-15 05:09:08 [default_loader.py:430] Loading weights took 13.03 seconds | +| sampler.model_loaded | 05:09:08.762 | 250.6 | 14.956827 | container log: (EngineCore pid=169) INFO 10-15 05:09:08 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.956827 seconds | +| sampler.compile_done | 05:09:45.574 | 287.4 | 35.82 | container log: (EngineCore pid=169) INFO 10-15 05:09:45 [monitor.py:53] torch.compile took 35.82 s in total | +| sampler.graphs_captured | 05:10:06.072 | 307.9 | 17.0 | container log: (EngineCore pid=169) INFO 10-15 05:10:06 [model_runner.py:1066] Graph capturing finished in 17 secs, took 0.78 GiB | +| sampler.kv_cache_sized | 05:10:06.701 | 308.6 | | container log: (EngineCore pid=169) INFO 10-15 05:10:06 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB | +| sampler.engine_init_done | 05:10:25.431 | 327.3 | 76.66 | container log: (EngineCore pid=169) INFO 10-15 05:10:25 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.66 s | +| sampler.ready_key | 05:10:26.000 | 327.9 | | redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 05:04:58.138 | -0.0 | +| client.create_model.start | 05:04:58.139 | 0.0 | +| client.create_model.done | 05:06:49.594 | 111.5 | +| client.forward_backward.start | 05:06:52.784 | 114.6 | +| client.forward_backward.done | 05:06:55.595 | 117.5 | +| client.optim_step.done | 05:06:57.617 | 119.5 | +| client.save_weights_and_get_sampling_client.start | 05:06:57.618 | 119.5 | +| client.save_weights_and_get_sampling_client.done | 05:06:59.533 | 121.4 | +| client.sample.start | 05:06:59.534 | 121.4 | +| client.sample.done | 05:10:27.431 | 329.3 | +| client.end | 05:10:27.434 | 329.3 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.9 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 90.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.1 | +| weights load | trainer.load_start | trainer.load_done | 13.0 | +| ready key | trainer.load_done | trainer.ready_key | -0.1 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 111.5 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.8 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.4 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 90.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 7.5 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 17.5 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 14.0 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.5 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.9 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.6 | +| client: first sample blocked | client.sample.start | client.sample.done | 207.9 | + diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt new file mode 100644 index 00000000..9c360d9c --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T05:04:53.958+00:00 diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt new file mode 100644 index 00000000..a9917932 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json new file mode 100644 index 00000000..22c659c3 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:57Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040908682239008", + "uid": "3920e58e-f415-4c96-8485-77a8cd538872" + }, + "spec": { + "accelerator": { + "memory": "29Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "44791173-6b7c-4d22-9a8b-46ef83190384", + "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:08:28Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "29Gi", + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:58Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040790072463004", + "uid": "125fac70-8359-4f66-bcb5-20df4c1e1764" + }, + "spec": { + "accelerator": { + "memory": "20Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "371ea4b9-ae90-4e15-a7b1-b34802637e23", + "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:06:30Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-125fac70", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "20Gi", + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log new file mode 100644 index 00000000..2b3010d6 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log @@ -0,0 +1,35 @@ +watching; recorders [18410, 18411, 18413, 18415, 18417, 18412, 18414, 18416] write into runs/2026-10-15_cold_start/after-D1-8b-h100-cached +collected 2 worker log(s) into runs/2026-10-15_cold_start/after-D1-8b-h100-cached +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.8 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 3.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.5 | +| weights load | trainer.load_start | trainer.load_done | 13.5 | +| ready key | trainer.load_done | trainer.ready_key | -27.0 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 25.3 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.4 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 15.4 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 9.5 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 12.9 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.6 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.3 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.2 | +| client: first sample blocked | client.sample.start | client.sample.done | 119.4 | + diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log new file mode 100644 index 00000000..43b56b1e --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log @@ -0,0 +1,32 @@ +2026-10-15T05:11:45.416+00:00 2026-10-15T05:11:45.437391305Z INFO: 127.0.0.1:34326 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T05:11:45.652+00:00 2026-10-15T05:11:45.675793966Z INFO: 127.0.0.1:34340 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T05:11:45.721+00:00 2026-10-15T05:11:45.743584668Z INFO: 127.0.0.1:34340 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T05:11:45.982+00:00 2026-10-15T05:11:46.006037569Z [API_SERVER] enqueued op=create_model request_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:11:45.984+00:00 2026-10-15T05:11:46.008026346Z INFO: 127.0.0.1:34350 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T05:11:50.728+00:00 2026-10-15T05:11:50.746569548Z INFO: 10.32.3.1:39220 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:11:55.956+00:00 2026-10-15T05:11:55.977933873Z INFO: 127.0.0.1:58848 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:12:00.725+00:00 2026-10-15T05:12:00.746355493Z INFO: 10.32.3.1:44030 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:12:10.663+00:00 2026-10-15T05:12:10.683497651Z INFO: 127.0.0.1:51194 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T05:12:10.723+00:00 2026-10-15T05:12:10.747036803Z INFO: 10.32.3.1:38468 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:12:13.399+00:00 2026-10-15T05:12:13.422272335Z [API_SERVER] enqueued op=forward_backward request_id=641d3bab-a7c0-4047-b609-7ca88e07767a model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:12:13.399+00:00 2026-10-15T05:12:13.423605880Z INFO: 127.0.0.1:51194 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T05:12:16.334+00:00 2026-10-15T05:12:16.355910312Z [API_SERVER] enqueued op=optim_step request_id=cb0f1a0a-f9af-4b9d-9aeb-e82e91a14640 model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:12:16.335+00:00 2026-10-15T05:12:16.356438363Z INFO: 127.0.0.1:51194 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T05:12:18.459+00:00 2026-10-15T05:12:18.478776460Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=bfeec1f4-669e-4b5b-b8c0-b876b9cae9dc model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:12:18.459+00:00 2026-10-15T05:12:18.479377759Z INFO: 127.0.0.1:51194 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T05:12:20.456+00:00 2026-10-15T05:12:20.479018182Z INFO: 127.0.0.1:35222 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T05:12:20.723+00:00 2026-10-15T05:12:20.747005620Z INFO: 10.32.3.1:38880 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:12:30.725+00:00 2026-10-15T05:12:30.747249473Z INFO: 10.32.3.1:33044 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:12:40.724+00:00 2026-10-15T05:12:40.746273596Z INFO: 10.32.3.1:41864 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:12:50.725+00:00 2026-10-15T05:12:50.746808349Z INFO: 10.32.3.1:41618 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:13:00.731+00:00 2026-10-15T05:13:00.748054823Z INFO: 10.32.3.1:46394 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:13:06.154+00:00 2026-10-15T05:13:06.171189246Z INFO: 127.0.0.1:44254 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:13:10.724+00:00 2026-10-15T05:13:10.747443779Z INFO: 10.32.3.1:38328 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:13:20.726+00:00 2026-10-15T05:13:20.747144519Z INFO: 10.32.3.1:44212 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:13:30.727+00:00 2026-10-15T05:13:30.746988779Z INFO: 10.32.3.1:39326 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:13:40.725+00:00 2026-10-15T05:13:40.747152571Z INFO: 10.32.3.1:53070 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:13:50.726+00:00 2026-10-15T05:13:50.746993399Z INFO: 10.32.3.1:51424 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:13:56.352+00:00 2026-10-15T05:13:56.369899995Z INFO: 127.0.0.1:35362 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:14:00.724+00:00 2026-10-15T05:14:00.747118474Z INFO: 10.32.3.1:46958 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:14:10.725+00:00 2026-10-15T05:14:10.747141725Z INFO: 10.32.3.1:35928 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:14:20.724+00:00 2026-10-15T05:14:20.746773992Z INFO: 10.32.3.1:39268 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl new file mode 100644 index 00000000..a1728906 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T05:11:45.156+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T05:11:45.157+00:00", "t": 0.001, "phase": "create_model.start"} +{"ts": "2026-10-15T05:12:10.447+00:00", "t": 25.291, "phase": "create_model.done", "model_id": "428ca04c-2ae6-41ed-9a4b-94f1cd79c97e"} +{"ts": "2026-10-15T05:12:13.323+00:00", "t": 28.167, "phase": "forward_backward.start"} +{"ts": "2026-10-15T05:12:16.232+00:00", "t": 31.076, "phase": "forward_backward.done"} +{"ts": "2026-10-15T05:12:18.352+00:00", "t": 33.196, "phase": "optim_step.done"} +{"ts": "2026-10-15T05:12:18.353+00:00", "t": 33.197, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T05:12:20.273+00:00", "t": 35.117, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T05:12:20.275+00:00", "t": 35.118, "phase": "sample.start"} +{"ts": "2026-10-15T05:14:19.688+00:00", "t": 154.532, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} +{"ts": "2026-10-15T05:14:19.689+00:00", "t": 154.533, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log new file mode 100644 index 00000000..832f9dbe --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.001s create_model.start +[cold-start] 25.291s create_model.done {'model_id': '428ca04c-2ae6-41ed-9a4b-94f1cd79c97e'} +[cold-start] 28.167s forward_backward.start +[cold-start] 31.076s forward_backward.done +[cold-start] 33.196s optim_step.done +[cold-start] 33.197s save_weights_and_get_sampling_client.start +[cold-start] 35.117s save_weights_and_get_sampling_client.done +[cold-start] 35.118s sample.start +[cold-start] 154.532s sample.done {'text': ' 4\n\nQuestion: What is '} +[cold-start] 154.533s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt new file mode 100644 index 00000000..dabd9d84 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T05:14:20.211+00:00 diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json new file mode 100644 index 00000000..4e274608 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json @@ -0,0 +1,2743 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:10Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:10Z", + "name": "cold-start-fio.18de9a0ef295773d", + "namespace": "openrl-system", + "resourceVersion": "1792039090760383011", + "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5be63862", + "namespace": "openrl-system", + "resourceVersion": "1792039092515007000", + "uid": "355b59b9-3661-499d-8640-3f40c3c89375" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5ecfa51a", + "namespace": "openrl-system", + "resourceVersion": "1792039092561663021", + "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f61ddd1e4", + "namespace": "openrl-system", + "resourceVersion": "1792039092605935011", + "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:40:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:40:41Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:40:41Z", + "name": "cold-start-fio.18de9a32081cf908", + "namespace": "openrl-system", + "resourceVersion": "1792039241446287009", + "uid": "14a62e42-13ba-4526-a04d-a934702e1163" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:18Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:18Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:18Z", + "name": "cold-start-fio.18de9a3a9867ce9f", + "namespace": "openrl-system", + "resourceVersion": "1792039278208207001", + "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad2bcac91", + "namespace": "openrl-system", + "resourceVersion": "1792039279188751014", + "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad378cd74", + "namespace": "openrl-system", + "resourceVersion": "1792039279219679006", + "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad5aed221", + "namespace": "openrl-system", + "resourceVersion": "1792039279235135013", + "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:11Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:11Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:11Z", + "name": "cold-start-fio.18de9a70cd2c6a68", + "namespace": "openrl-system", + "resourceVersion": "1792039511048479001", + "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "resourceVersion": "1791941114537823021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Stopping container api-server", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", + "namespace": "openrl-system", + "resourceVersion": "1792039545418735013", + "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "namespace": "openrl-system", + "resourceVersion": "1792039545300351007", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", + "namespace": "openrl-system", + "resourceVersion": "1792039545387567001", + "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" + }, + "reason": "SuccessfulDelete", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546048591003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", + "namespace": "openrl-system", + "resourceVersion": "1792039546102351012", + "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:49Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:49Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", + "namespace": "openrl-system", + "resourceVersion": "1792039549712335019", + "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", + "namespace": "openrl-system", + "resourceVersion": "1792039558721215017", + "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", + "namespace": "openrl-system", + "resourceVersion": "1792039558783199013", + "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", + "namespace": "openrl-system", + "resourceVersion": "1792039558862303009", + "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "namespace": "openrl-system", + "resourceVersion": "1792039545940671018", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", + "namespace": "openrl-system", + "resourceVersion": "1792039546068975009", + "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" + }, + "reason": "SuccessfulCreate", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545232655018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78c9668cf7", + "namespace": "openrl-system", + "resourceVersion": "1792039545327119005", + "uid": "7b1433df-3524-478b-aba2-8979950e245e" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545464319018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78ef8b9e1b", + "namespace": "openrl-system", + "resourceVersion": "1792039545956079020", + "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:15Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875490623001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:15Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", + "namespace": "openrl-system", + "resourceVersion": "1792039875609279013", + "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:19Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:19Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", + "namespace": "openrl-system", + "resourceVersion": "1792039879126319008", + "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", + "namespace": "openrl-system", + "resourceVersion": "1792040061645247020", + "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", + "namespace": "openrl-system", + "resourceVersion": "1792040061849391002", + "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", + "namespace": "openrl-system", + "resourceVersion": "1792040062252079023", + "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", + "namespace": "openrl-system", + "resourceVersion": "1792040192780783021", + "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264403887001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", + "namespace": "openrl-system", + "resourceVersion": "1792040264444463021", + "uid": "a85006a2-68a6-4919-b148-08d58b546d07" + }, + "reason": "FailedScheduling", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Warning" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264434463001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", + "namespace": "openrl-system", + "resourceVersion": "1792040264884223002", + "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" + }, + "reason": "NotTriggerScaleUp", + "reportingComponent": "cluster-autoscaler", + "reportingInstance": "", + "source": { + "component": "cluster-autoscaler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264893039001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", + "namespace": "openrl-system", + "resourceVersion": "1792040293171567001", + "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", + "namespace": "openrl-system", + "resourceVersion": "1792040296672127014", + "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", + "namespace": "openrl-system", + "resourceVersion": "1792040296703007017", + "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", + "namespace": "openrl-system", + "resourceVersion": "1792040296889487014", + "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", + "namespace": "openrl-system", + "resourceVersion": "1792040427147711019", + "uid": "cc4da270-def2-456f-9374-372be8c717e4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:38Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498635999001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:38Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:38Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", + "namespace": "openrl-system", + "resourceVersion": "1792040498767471000", + "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", + "namespace": "openrl-system", + "resourceVersion": "1792040502270255017", + "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", + "namespace": "openrl-system", + "resourceVersion": "1792040502296943006", + "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", + "namespace": "openrl-system", + "resourceVersion": "1792040502475407023", + "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:09Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:09Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:09Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", + "namespace": "openrl-system", + "resourceVersion": "1792040649924575012", + "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:52Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672220015012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:52Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", + "namespace": "openrl-system", + "resourceVersion": "1792039672350591019", + "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:56Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:56Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:56Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", + "namespace": "openrl-system", + "resourceVersion": "1792039676166239015", + "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", + "namespace": "openrl-system", + "resourceVersion": "1792039844457167000", + "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", + "namespace": "openrl-system", + "resourceVersion": "1792039844632591021", + "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:45Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:45Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", + "namespace": "openrl-system", + "resourceVersion": "1792039845170959016", + "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", + "namespace": "openrl-system", + "resourceVersion": "1792040192981183013", + "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:23Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040242926223012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:23Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:23Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", + "namespace": "openrl-system", + "resourceVersion": "1792040243043743024", + "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", + "namespace": "openrl-system", + "resourceVersion": "1792040246545487023", + "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", + "namespace": "openrl-system", + "resourceVersion": "1792040246576895006", + "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", + "namespace": "openrl-system", + "resourceVersion": "1792040246742815002", + "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", + "namespace": "openrl-system", + "resourceVersion": "1792040427285631023", + "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:16Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476805407012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:16Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", + "namespace": "openrl-system", + "resourceVersion": "1792040476942623014", + "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", + "namespace": "openrl-system", + "resourceVersion": "1792040480747055005", + "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", + "namespace": "openrl-system", + "resourceVersion": "1792040480787967019", + "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", + "namespace": "openrl-system", + "resourceVersion": "1792040480963103007", + "uid": "d3158913-c79e-43d4-b468-d748641d49fe" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:10Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:10Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", + "namespace": "openrl-system", + "resourceVersion": "1792040650191935010", + "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", + "namespace": "openrl-system", + "resourceVersion": "1792038835732623016", + "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:58Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818213471023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:58Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:58Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", + "namespace": "openrl-system", + "resourceVersion": "1792040818411407005", + "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:07:01Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:07:01Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:07:01Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", + "namespace": "openrl-system", + "resourceVersion": "1792040821861695012", + "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", + "namespace": "openrl-system", + "resourceVersion": "1792040908347887024", + "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", + "namespace": "openrl-system", + "resourceVersion": "1792040908367439012", + "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", + "namespace": "openrl-system", + "resourceVersion": "1792040908418671004", + "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:10:57Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:10:57Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:10:57Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bd8cbffc810", + "namespace": "openrl-system", + "resourceVersion": "1792041057212047016", + "uid": "4a67fe5e-01b5-4b84-9453-b20887271ae4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:18Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138729647023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:18Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:18Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bebcffc9f4c", + "namespace": "openrl-system", + "resourceVersion": "1792041138871999014", + "uid": "3e19d3e2-4122-4f9d-8324-8f74961c36bc" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9c8a8459", + "namespace": "openrl-system", + "resourceVersion": "1792041142294159019", + "uid": "aaaf55ef-ec50-4fa5-b6a7-28a27d836f8e" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9d08793e", + "namespace": "openrl-system", + "resourceVersion": "1792041142311631019", + "uid": "5ab187bd-863a-4013-9e2d-185b71c68ead" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9beca1f20c38", + "namespace": "openrl-system", + "resourceVersion": "1792041142412303013", + "uid": "193a6a12-3e5a-44e3-8d69-ea9d46ee4707" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", + "namespace": "openrl-system", + "resourceVersion": "1792038835415695012", + "uid": "918abc9a-1f24-4767-8928-b202482f11ed" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:59Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699253647006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:59Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:59Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", + "namespace": "openrl-system", + "resourceVersion": "1792040699411615020", + "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:05:02Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:05:02Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:05:02Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", + "namespace": "openrl-system", + "resourceVersion": "1792040702849391020", + "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", + "namespace": "openrl-system", + "resourceVersion": "1792040789347727015", + "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", + "namespace": "openrl-system", + "resourceVersion": "1792040789380863020", + "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", + "namespace": "openrl-system", + "resourceVersion": "1792040789417615006", + "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:10:57Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:10:57Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:10:57Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9bd8d38751ab", + "namespace": "openrl-system", + "resourceVersion": "1792041057319999002", + "uid": "0af06e74-e0c1-44de-a612-6fba47fb2b5c" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106258495006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:46Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:46Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be43ff3ded1", + "namespace": "openrl-system", + "resourceVersion": "1792041106381007003", + "uid": "4842f6c0-d606-444b-a970-6e9484189707" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50d27add6", + "namespace": "openrl-system", + "resourceVersion": "1792041109821407010", + "uid": "4e9749a9-e70a-453f-b51f-4d5ea7a60e86" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50daf2189", + "namespace": "openrl-system", + "resourceVersion": "1792041109840015005", + "uid": "2950a976-a558-471f-94fb-1ac680004e4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be5134afada", + "namespace": "openrl-system", + "resourceVersion": "1792041109931471020", + "uid": "d1c0b2ae-50a0-41e8-8333-441dd2250b23" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt new file mode 100644 index 00000000..f721cff7 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt @@ -0,0 +1,222 @@ +2026-10-15T05:12:37.408864742Z INFO 10-15 05:12:37 [model.py:692] Resolved architecture: Qwen3ForCausalLM +2026-10-15T05:12:37.409029510Z INFO 10-15 05:12:37 [model.py:2030] Using max model len 8192 +2026-10-15T05:12:37.410626201Z INFO 10-15 05:12:37 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. +2026-10-15T05:12:37.411444983Z INFO 10-15 05:12:37 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T05:12:39.815686755Z WARNING 10-15 05:12:39 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T05:12:46.943669994Z (EngineCore pid=169) INFO 10-15 05:12:46 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T05:12:47.276765140Z (EngineCore pid=169) INFO 10-15 05:12:47 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_d8308236b8a9489688b723c3383c2fc7 backend=nccl +2026-10-15T05:12:47.281242229Z (EngineCore pid=169) INFO 10-15 05:12:47 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T05:12:47.281272727Z (EngineCore pid=169) INFO 10-15 05:12:47 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T05:12:47.997459494Z (EngineCore pid=169) INFO 10-15 05:12:47 [model_runner.py:396] Loading model from scratch... +2026-10-15T05:12:48.572994219Z (EngineCore pid=169) INFO 10-15 05:12:48 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T05:12:48.573332535Z (EngineCore pid=169) INFO 10-15 05:12:48 [flash_attn.py:1116] Using FlashAttention version 3 +2026-10-15T05:12:48.688784183Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T05:12:48.689006740Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T05:12:48.986411683Z (EngineCore pid=169) INFO 10-15 05:12:48 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. +2026-10-15T05:12:48.986441106Z (EngineCore pid=169) INFO 10-15 05:12:48 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T05:12:48.987162311Z (EngineCore pid=169) +Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T05:11:54.307386428Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T05:11:54.307409055Z +2026-10-15T05:11:54.511196464Z Loading base model Qwen/Qwen3-8B to cuda:0... +2026-10-15T05:11:54.620348814Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T05:11:55.488744741Z [prefetch] reading 5 file(s), 15.26 GiB, from nfs into the page cache with 8 threads +2026-10-15T05:12:05.401433205Z [prefetch] done: 15.26 GiB in 9.9 s (1576 MiB/s) +2026-10-15T05:12:07.881306965Z +Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json new file mode 100644 index 00000000..7a668937 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "generateName": "open-rl-api-server-6d86b498f5-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "6d86b498f5", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + } + ], + "resourceVersion": "1792039570790271003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2ns79", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:59Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:45:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.11", + "podIPs": [ + { + "ip": "10.32.3.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:45:46Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:18Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-sampler", + "uid": "996528ae-d593-4922-86b8-ea1303243615" + } + ], + "resourceVersion": "1792041143127711023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-sampler-996528ae" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "20f5c64d-1657-4c78-92ee-24fa609f498b" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-wg5z6", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-wg5z6", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:12:23Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:12:18Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:12:23Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:12:23Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:12:18Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "40Gi" + }, + "containerID": "containerd://407aa25fc65fc92e4c27342d67e1ed826acd343dc2c6f41a01d7e922266f8d3a", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:12:22Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-wg5z6", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.1.10", + "podIPs": [ + { + "ip": "10.32.1.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:12:18Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:46Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-trainer", + "uid": "e9ddbd1c-f7f3-431b-abf2-f9f7cbdac473" + } + ], + "resourceVersion": "1792041110653471006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "9b290f43-cc99-4ea2-b399-4c09027c74d4" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-n8w58", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-n8w58", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:11:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:11:46Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:11:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:11:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:11:46Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "36Gi" + }, + "containerID": "containerd://fb935263c9491ef19bf5a85a25c2c6c1f3f99095042469eaec8e63809876f6da", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:11:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-n8w58", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.4.11", + "podIPs": [ + { + "ip": "10.32.4.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:11:46Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log new file mode 100644 index 00000000..4eb3d4ac --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log @@ -0,0 +1,342 @@ +2026-10-15T05:11:41.877+00:00 2026-10-15T05:11:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:41.878+00:00 2026-10-15T05:11:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:42.884+00:00 2026-10-15T05:11:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:42.884+00:00 2026-10-15T05:11:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:43.893+00:00 2026-10-15T05:11:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:43.893+00:00 2026-10-15T05:11:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:44.894+00:00 2026-10-15T05:11:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:44.894+00:00 2026-10-15T05:11:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:45.899+00:00 2026-10-15T05:11:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:45.899+00:00 2026-10-15T05:11:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:46.917+00:00 2026-10-15T05:11:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:46.917+00:00 2026-10-15T05:11:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:47.910+00:00 2026-10-15T05:11:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:47.910+00:00 2026-10-15T05:11:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:48.918+00:00 2026-10-15T05:11:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:48.918+00:00 2026-10-15T05:11:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:49.921+00:00 2026-10-15T05:11:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:49.922+00:00 2026-10-15T05:11:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:50.927+00:00 2026-10-15T05:11:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:50.927+00:00 2026-10-15T05:11:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:51.932+00:00 2026-10-15T05:11:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:51.932+00:00 2026-10-15T05:11:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:52.938+00:00 2026-10-15T05:11:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:52.938+00:00 2026-10-15T05:11:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:53.946+00:00 2026-10-15T05:11:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:53.946+00:00 2026-10-15T05:11:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:54.948+00:00 2026-10-15T05:11:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:54.949+00:00 2026-10-15T05:11:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:55.952+00:00 2026-10-15T05:11:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:55.954+00:00 2026-10-15T05:11:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:56.958+00:00 2026-10-15T05:11:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:56.958+00:00 2026-10-15T05:11:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:57.963+00:00 2026-10-15T05:11:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:57.963+00:00 2026-10-15T05:11:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:59.058+00:00 2026-10-15T05:11:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:59.058+00:00 2026-10-15T05:11:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:11:59.975+00:00 2026-10-15T05:11:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:11:59.975+00:00 2026-10-15T05:11:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:00.979+00:00 2026-10-15T05:12:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:00.979+00:00 2026-10-15T05:12:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:01.983+00:00 2026-10-15T05:12:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:01.983+00:00 2026-10-15T05:12:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:02.988+00:00 2026-10-15T05:12:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:02.988+00:00 2026-10-15T05:12:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:03.993+00:00 2026-10-15T05:12:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:03.993+00:00 2026-10-15T05:12:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:04.999+00:00 2026-10-15T05:12:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:04.999+00:00 2026-10-15T05:12:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:06.004+00:00 2026-10-15T05:12:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:06.004+00:00 2026-10-15T05:12:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:07.010+00:00 2026-10-15T05:12:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:07.010+00:00 2026-10-15T05:12:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:08.013+00:00 2026-10-15T05:12:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:08.013+00:00 2026-10-15T05:12:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:09.025+00:00 2026-10-15T05:12:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:09.025+00:00 2026-10-15T05:12:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:10.023+00:00 2026-10-15T05:12:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:10.023+00:00 2026-10-15T05:12:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:11.028+00:00 2026-10-15T05:12:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:11.028+00:00 2026-10-15T05:12:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:12.035+00:00 2026-10-15T05:12:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:12.035+00:00 2026-10-15T05:12:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:13.040+00:00 2026-10-15T05:12:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:13.040+00:00 2026-10-15T05:12:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:14.046+00:00 2026-10-15T05:12:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:14.046+00:00 2026-10-15T05:12:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:15.050+00:00 2026-10-15T05:12:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:15.050+00:00 2026-10-15T05:12:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:16.056+00:00 2026-10-15T05:12:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:16.056+00:00 2026-10-15T05:12:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:17.059+00:00 2026-10-15T05:12:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:17.059+00:00 2026-10-15T05:12:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:18.064+00:00 2026-10-15T05:12:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:18.064+00:00 2026-10-15T05:12:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:19.069+00:00 2026-10-15T05:12:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:19.069+00:00 2026-10-15T05:12:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:20.074+00:00 2026-10-15T05:12:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:20.074+00:00 2026-10-15T05:12:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:21.079+00:00 2026-10-15T05:12:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:21.079+00:00 2026-10-15T05:12:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:22.086+00:00 2026-10-15T05:12:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:22.086+00:00 2026-10-15T05:12:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:23.091+00:00 2026-10-15T05:12:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:23.091+00:00 2026-10-15T05:12:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:24.097+00:00 2026-10-15T05:12:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:24.097+00:00 2026-10-15T05:12:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:25.101+00:00 2026-10-15T05:12:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:25.101+00:00 2026-10-15T05:12:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:26.106+00:00 2026-10-15T05:12:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:26.106+00:00 2026-10-15T05:12:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:27.112+00:00 2026-10-15T05:12:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:27.112+00:00 2026-10-15T05:12:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:28.121+00:00 2026-10-15T05:12:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:28.121+00:00 2026-10-15T05:12:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:29.121+00:00 2026-10-15T05:12:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:29.121+00:00 2026-10-15T05:12:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:30.130+00:00 2026-10-15T05:12:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:30.131+00:00 2026-10-15T05:12:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:31.131+00:00 2026-10-15T05:12:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:31.131+00:00 2026-10-15T05:12:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:32.136+00:00 2026-10-15T05:12:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:32.136+00:00 2026-10-15T05:12:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:33.141+00:00 2026-10-15T05:12:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:33.141+00:00 2026-10-15T05:12:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:34.146+00:00 2026-10-15T05:12:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:34.146+00:00 2026-10-15T05:12:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:35.150+00:00 2026-10-15T05:12:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:35.151+00:00 2026-10-15T05:12:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:36.156+00:00 2026-10-15T05:12:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:36.156+00:00 2026-10-15T05:12:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:37.160+00:00 2026-10-15T05:12:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:37.160+00:00 2026-10-15T05:12:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:38.166+00:00 2026-10-15T05:12:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:38.166+00:00 2026-10-15T05:12:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:39.170+00:00 2026-10-15T05:12:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:39.171+00:00 2026-10-15T05:12:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:40.176+00:00 2026-10-15T05:12:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:40.177+00:00 2026-10-15T05:12:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:41.181+00:00 2026-10-15T05:12:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:41.182+00:00 2026-10-15T05:12:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:42.186+00:00 2026-10-15T05:12:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:42.186+00:00 2026-10-15T05:12:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:43.191+00:00 2026-10-15T05:12:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:43.191+00:00 2026-10-15T05:12:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:44.197+00:00 2026-10-15T05:12:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:44.197+00:00 2026-10-15T05:12:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:45.202+00:00 2026-10-15T05:12:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:45.203+00:00 2026-10-15T05:12:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:46.210+00:00 2026-10-15T05:12:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:46.210+00:00 2026-10-15T05:12:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:47.214+00:00 2026-10-15T05:12:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:47.214+00:00 2026-10-15T05:12:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:48.220+00:00 2026-10-15T05:12:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:48.220+00:00 2026-10-15T05:12:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:49.224+00:00 2026-10-15T05:12:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:49.224+00:00 2026-10-15T05:12:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:50.261+00:00 2026-10-15T05:12:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:50.261+00:00 2026-10-15T05:12:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:51.235+00:00 2026-10-15T05:12:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:51.235+00:00 2026-10-15T05:12:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:52.240+00:00 2026-10-15T05:12:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:52.240+00:00 2026-10-15T05:12:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:53.246+00:00 2026-10-15T05:12:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:53.246+00:00 2026-10-15T05:12:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:54.251+00:00 2026-10-15T05:12:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:54.251+00:00 2026-10-15T05:12:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:55.256+00:00 2026-10-15T05:12:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:55.256+00:00 2026-10-15T05:12:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:56.262+00:00 2026-10-15T05:12:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:56.262+00:00 2026-10-15T05:12:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:57.267+00:00 2026-10-15T05:12:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:57.267+00:00 2026-10-15T05:12:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:58.272+00:00 2026-10-15T05:12:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:58.272+00:00 2026-10-15T05:12:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:12:59.282+00:00 2026-10-15T05:12:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:12:59.282+00:00 2026-10-15T05:12:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:00.282+00:00 2026-10-15T05:13:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:00.282+00:00 2026-10-15T05:13:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:01.288+00:00 2026-10-15T05:13:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:01.288+00:00 2026-10-15T05:13:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:02.293+00:00 2026-10-15T05:13:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:02.293+00:00 2026-10-15T05:13:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:03.298+00:00 2026-10-15T05:13:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:03.298+00:00 2026-10-15T05:13:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:04.305+00:00 2026-10-15T05:13:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:04.305+00:00 2026-10-15T05:13:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:05.311+00:00 2026-10-15T05:13:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:05.311+00:00 2026-10-15T05:13:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:06.315+00:00 2026-10-15T05:13:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:06.315+00:00 2026-10-15T05:13:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:07.320+00:00 2026-10-15T05:13:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:07.321+00:00 2026-10-15T05:13:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:08.325+00:00 2026-10-15T05:13:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:08.325+00:00 2026-10-15T05:13:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:09.331+00:00 2026-10-15T05:13:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:09.332+00:00 2026-10-15T05:13:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:10.344+00:00 2026-10-15T05:13:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:10.344+00:00 2026-10-15T05:13:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:11.378+00:00 2026-10-15T05:13:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:11.378+00:00 2026-10-15T05:13:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:12.350+00:00 2026-10-15T05:13:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:12.350+00:00 2026-10-15T05:13:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:13.355+00:00 2026-10-15T05:13:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:13.355+00:00 2026-10-15T05:13:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:14.360+00:00 2026-10-15T05:13:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:14.360+00:00 2026-10-15T05:13:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:15.366+00:00 2026-10-15T05:13:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:15.366+00:00 2026-10-15T05:13:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:16.371+00:00 2026-10-15T05:13:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:16.371+00:00 2026-10-15T05:13:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:17.376+00:00 2026-10-15T05:13:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:17.376+00:00 2026-10-15T05:13:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:18.382+00:00 2026-10-15T05:13:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:18.382+00:00 2026-10-15T05:13:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:19.386+00:00 2026-10-15T05:13:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:19.387+00:00 2026-10-15T05:13:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:20.393+00:00 2026-10-15T05:13:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:20.393+00:00 2026-10-15T05:13:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:21.616+00:00 2026-10-15T05:13:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:21.616+00:00 2026-10-15T05:13:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:22.404+00:00 2026-10-15T05:13:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:22.404+00:00 2026-10-15T05:13:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:23.408+00:00 2026-10-15T05:13:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:23.408+00:00 2026-10-15T05:13:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:24.412+00:00 2026-10-15T05:13:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:24.413+00:00 2026-10-15T05:13:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:25.417+00:00 2026-10-15T05:13:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:25.417+00:00 2026-10-15T05:13:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:26.424+00:00 2026-10-15T05:13:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:26.424+00:00 2026-10-15T05:13:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:27.435+00:00 2026-10-15T05:13:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:27.436+00:00 2026-10-15T05:13:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:28.436+00:00 2026-10-15T05:13:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:28.436+00:00 2026-10-15T05:13:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:29.441+00:00 2026-10-15T05:13:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:29.441+00:00 2026-10-15T05:13:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:30.451+00:00 2026-10-15T05:13:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:30.451+00:00 2026-10-15T05:13:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:31.452+00:00 2026-10-15T05:13:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:31.452+00:00 2026-10-15T05:13:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:32.463+00:00 2026-10-15T05:13:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:32.463+00:00 2026-10-15T05:13:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:33.462+00:00 2026-10-15T05:13:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:33.462+00:00 2026-10-15T05:13:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:34.467+00:00 2026-10-15T05:13:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:34.467+00:00 2026-10-15T05:13:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:35.473+00:00 2026-10-15T05:13:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:35.473+00:00 2026-10-15T05:13:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:36.478+00:00 2026-10-15T05:13:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:36.478+00:00 2026-10-15T05:13:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:37.483+00:00 2026-10-15T05:13:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:37.483+00:00 2026-10-15T05:13:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:38.488+00:00 2026-10-15T05:13:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:38.489+00:00 2026-10-15T05:13:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:39.507+00:00 2026-10-15T05:13:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:39.507+00:00 2026-10-15T05:13:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:40.499+00:00 2026-10-15T05:13:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:40.499+00:00 2026-10-15T05:13:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:41.507+00:00 2026-10-15T05:13:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:41.507+00:00 2026-10-15T05:13:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:42.511+00:00 2026-10-15T05:13:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:42.511+00:00 2026-10-15T05:13:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:43.516+00:00 2026-10-15T05:13:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:43.516+00:00 2026-10-15T05:13:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:44.521+00:00 2026-10-15T05:13:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:44.521+00:00 2026-10-15T05:13:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:45.526+00:00 2026-10-15T05:13:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:45.526+00:00 2026-10-15T05:13:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:46.531+00:00 2026-10-15T05:13:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:46.532+00:00 2026-10-15T05:13:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:47.537+00:00 2026-10-15T05:13:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:47.537+00:00 2026-10-15T05:13:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:48.542+00:00 2026-10-15T05:13:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:48.542+00:00 2026-10-15T05:13:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:49.549+00:00 2026-10-15T05:13:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:49.549+00:00 2026-10-15T05:13:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:50.554+00:00 2026-10-15T05:13:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:50.554+00:00 2026-10-15T05:13:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:51.560+00:00 2026-10-15T05:13:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:51.560+00:00 2026-10-15T05:13:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:52.565+00:00 2026-10-15T05:13:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:52.565+00:00 2026-10-15T05:13:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:53.571+00:00 2026-10-15T05:13:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:53.571+00:00 2026-10-15T05:13:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:54.578+00:00 2026-10-15T05:13:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:54.578+00:00 2026-10-15T05:13:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:55.580+00:00 2026-10-15T05:13:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:55.580+00:00 2026-10-15T05:13:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:56.585+00:00 2026-10-15T05:13:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:56.586+00:00 2026-10-15T05:13:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:57.590+00:00 2026-10-15T05:13:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:57.591+00:00 2026-10-15T05:13:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:58.609+00:00 2026-10-15T05:13:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:58.609+00:00 2026-10-15T05:13:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:13:59.601+00:00 2026-10-15T05:13:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:13:59.601+00:00 2026-10-15T05:13:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:00.606+00:00 2026-10-15T05:14:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:00.607+00:00 2026-10-15T05:14:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:01.612+00:00 2026-10-15T05:14:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:01.612+00:00 2026-10-15T05:14:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:02.619+00:00 2026-10-15T05:14:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:02.619+00:00 2026-10-15T05:14:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:03.623+00:00 2026-10-15T05:14:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:03.623+00:00 2026-10-15T05:14:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:04.628+00:00 2026-10-15T05:14:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:04.628+00:00 2026-10-15T05:14:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:05.639+00:00 2026-10-15T05:14:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:05.639+00:00 2026-10-15T05:14:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:06.638+00:00 2026-10-15T05:14:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:06.639+00:00 2026-10-15T05:14:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:07.645+00:00 2026-10-15T05:14:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:07.645+00:00 2026-10-15T05:14:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:08.649+00:00 2026-10-15T05:14:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:08.649+00:00 2026-10-15T05:14:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:09.655+00:00 2026-10-15T05:14:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:09.656+00:00 2026-10-15T05:14:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:10.662+00:00 2026-10-15T05:14:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:10.662+00:00 2026-10-15T05:14:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:11.668+00:00 2026-10-15T05:14:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:11.668+00:00 2026-10-15T05:14:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:12.671+00:00 2026-10-15T05:14:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:12.671+00:00 2026-10-15T05:14:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:13.677+00:00 2026-10-15T05:14:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:13.677+00:00 2026-10-15T05:14:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:14.683+00:00 2026-10-15T05:14:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:14.684+00:00 2026-10-15T05:14:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:15.688+00:00 2026-10-15T05:14:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:15.688+00:00 2026-10-15T05:14:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:16.693+00:00 2026-10-15T05:14:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:16.693+00:00 2026-10-15T05:14:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:17.698+00:00 2026-10-15T05:14:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:17.698+00:00 2026-10-15T05:14:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:18.703+00:00 2026-10-15T05:14:18Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:18.703+00:00 2026-10-15T05:14:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:18.703+00:00 2026-10-15T05:14:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:19.707+00:00 2026-10-15T05:14:19Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:19.707+00:00 2026-10-15T05:14:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:19.707+00:00 2026-10-15T05:14:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:20.713+00:00 2026-10-15T05:14:20Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:20.713+00:00 2026-10-15T05:14:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:20.713+00:00 2026-10-15T05:14:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:21.718+00:00 2026-10-15T05:14:21Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:21.718+00:00 2026-10-15T05:14:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:21.718+00:00 2026-10-15T05:14:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:22.723+00:00 2026-10-15T05:14:22Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:22.723+00:00 2026-10-15T05:14:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:22.723+00:00 2026-10-15T05:14:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:23.729+00:00 2026-10-15T05:14:23Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:23.729+00:00 2026-10-15T05:14:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:23.729+00:00 2026-10-15T05:14:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:24.734+00:00 2026-10-15T05:14:24Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:24.735+00:00 2026-10-15T05:14:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:24.735+00:00 2026-10-15T05:14:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:25.742+00:00 2026-10-15T05:14:25Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:25.742+00:00 2026-10-15T05:14:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:25.742+00:00 2026-10-15T05:14:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:26.745+00:00 2026-10-15T05:14:26Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:26.745+00:00 2026-10-15T05:14:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:26.745+00:00 2026-10-15T05:14:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:14:27.787+00:00 2026-10-15T05:14:27Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:14:27.787+00:00 2026-10-15T05:14:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:14:27.787+00:00 2026-10-15T05:14:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log new file mode 100644 index 00000000..d52fd557 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T05:11:46.113+00:00 2026-10-15T05:11:46.136519288Z {"level":"info","ts":"2026-10-15T05:11:46Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"76087a4e-44f6-4134-908c-44b725c30029","claim":"claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} +2026-10-15T05:12:18.598+00:00 2026-10-15T05:12:18.620658575Z {"level":"info","ts":"2026-10-15T05:12:18Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"30427539-027e-42b8-833c-14c6aa57927a","claim":"claim-lora-qwen-qwen3-8b-0-sampler-996528ae","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json new file mode 100644 index 00000000..279002b2 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json @@ -0,0 +1,267 @@ +{ + "case": "after-D1-8b-h100-cached", + "collected": "2026-10-15T05:14:27.870+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 05:11:45.156000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen3-8B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 05:11:45.157000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 05:12:10.447000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "428ca04c-2ae6-41ed-9a4b-94f1cd79c97e" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 05:12:13.323000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 05:12:16.232000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 05:12:18.352000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 05:12:18.353000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 05:12:20.273000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 05:12:20.275000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 05:14:19.688000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4\n\nQuestion: What is " + }, + "client.end": { + "ts": "2026-10-15 05:14:19.689000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen3-8b-0-sampler", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "pod": "orw-lora-qwen-qwen3-8b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 05:12:18+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:12:18.877000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 05:12:18.877000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 05:12:18.877000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 05:12:23+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 05:12:18+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 05:12:18+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 05:12:23+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 05:12:23+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 05:12:18+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 05:12:22+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 05:12:18+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 05:12:22+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "sampler.event_Created": { + "ts": "2026-10-15 05:12:22+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 05:12:22+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 05:12:37.408864+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 05:12:46.943669+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:12:46 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 05:12:47.997459+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:12:47 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 05:12:48.986441+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:12:48 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 05:13:00.893700+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:00 [default_loader.py:430] Loading weights took 11.91 seconds", + "self_reported_s": 11.91 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 05:13:01.578989+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:01 [model_runner.py:428] Model loading took 18.17 GiB memory and 13.845580 seconds", + "self_reported_s": 13.84558 + }, + "sampler.compile_done": { + "ts": "2026-10-15 05:13:38.517327+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:38 [monitor.py:53] torch.compile took 35.91 s in total", + "self_reported_s": 35.91 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 05:13:58.672379+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:58 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", + "self_reported_s": 16.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 05:13:59.319915+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:59 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 05:14:17.795200+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:14:17 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.21 s", + "self_reported_s": 76.21 + }, + "sampler.ready_key": { + "ts": "2026-10-15 05:14:18+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + }, + "trainer": { + "workload": "lora-qwen-qwen3-8b-0-trainer", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "pod": "orw-lora-qwen-qwen3-8b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 05:11:45+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:11:45.752000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 05:11:45.752000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 05:11:47.402000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 05:11:50+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 05:11:46+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 05:11:46+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 05:11:50+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 05:11:50+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 05:11:46+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 05:11:49+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 05:11:46+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 05:11:49+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "trainer.event_Created": { + "ts": "2026-10-15 05:11:49+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 05:11:49+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 05:11:54.291598+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 05:11:54.511196+00:00", + "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 05:12:08.031266+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 05:12:08.033054+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 05:11:41+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.md b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.md new file mode 100644 index 00000000..23e7cad5 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.md @@ -0,0 +1,109 @@ +# Cold start timeline: after-D1-8b-h100-cached + +t=0 is the client's `create_model` call at 2026-10-15T05:11:45.157+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen3-8b-0-trainer` on `gke-open-rl-dra-h100-trainer-702095b4-1g3a` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.ready_key | 05:11:41.000 | -4.2 | | redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B | +| trainer.workload_created | 05:11:45.000 | -0.2 | | Workload creationTimestamp (apiserver) | +| trainer.workload_first_seen_by_poller | 05:11:45.752 | 0.6 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 05:11:45.752 | 0.6 | | workloads.tsv poller: status.podName set | +| trainer.pod_created | 05:11:46.000 | 0.8 | | pod creationTimestamp (apiserver) | +| trainer.pod_Initialized | 05:11:46.000 | 0.8 | | pod condition Initialized | +| trainer.pod_PodScheduled | 05:11:46.000 | 0.8 | | pod condition PodScheduled | +| trainer.event_Scheduled | 05:11:46.000 | 0.8 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a | +| trainer.workload_reports_node | 05:11:47.402 | 2.2 | | workloads.tsv poller: status.nodeName set | +| trainer.container_started | 05:11:49.000 | 3.8 | | containerStatuses.state.running.startedAt | +| trainer.event_Pulled | 05:11:49.000 | 3.8 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| trainer.event_Created | 05:11:49.000 | 3.8 | | event: Container created | +| trainer.event_Started | 05:11:49.000 | 3.8 | | event: Container started | +| trainer.condition_Placed_True | 05:11:50.000 | 4.8 | | Workload condition Placed | +| trainer.pod_Ready | 05:11:50.000 | 4.8 | | pod condition Ready | +| trainer.pod_ContainersReady | 05:11:50.000 | 4.8 | | pod condition ContainersReady | +| trainer.process_first_log | 05:11:54.291 | 9.1 | | container log, first line | +| trainer.load_start | 05:11:54.511 | 9.4 | | container log: Loading base model Qwen/Qwen3-8B to cuda:0... | +| trainer.load_done | 05:12:08.031 | 22.9 | | container log: Successfully loaded. | +| trainer.processor_started | 05:12:08.033 | 22.9 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1). | + +## sampler: `orw-lora-qwen-qwen3-8b-0-sampler` on `gke-open-rl-dra-h100-sampler-a721f127-0853` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_created | 05:12:18.000 | 32.8 | | Workload creationTimestamp (apiserver) | +| sampler.pod_created | 05:12:18.000 | 32.8 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 05:12:18.000 | 32.8 | | pod condition Initialized | +| sampler.pod_PodScheduled | 05:12:18.000 | 32.8 | | pod condition PodScheduled | +| sampler.event_Scheduled | 05:12:18.000 | 32.8 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853 | +| sampler.workload_first_seen_by_poller | 05:12:18.877 | 33.7 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_reports_pod | 05:12:18.877 | 33.7 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 05:12:18.877 | 33.7 | | workloads.tsv poller: status.nodeName set | +| sampler.container_started | 05:12:22.000 | 36.8 | | containerStatuses.state.running.startedAt | +| sampler.event_Pulled | 05:12:22.000 | 36.8 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| sampler.event_Created | 05:12:22.000 | 36.8 | | event: Container created | +| sampler.event_Started | 05:12:22.000 | 36.8 | | event: Container started | +| sampler.condition_Placed_True | 05:12:23.000 | 37.8 | | Workload condition Placed | +| sampler.pod_Ready | 05:12:23.000 | 37.8 | | pod condition Ready | +| sampler.pod_ContainersReady | 05:12:23.000 | 37.8 | | pod condition ContainersReady | +| sampler.process_first_log | 05:12:37.408 | 52.3 | | container log, first line | +| sampler.engine_init_start | 05:12:46.943 | 61.8 | | container log: (EngineCore pid=169) INFO 10-15 05:12:46 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 05:12:47.997 | 62.8 | | container log: (EngineCore pid=169) INFO 10-15 05:12:47 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 05:12:48.986 | 63.8 | | container log: (EngineCore pid=169) INFO 10-15 05:12:48 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 05:13:00.893 | 75.7 | 11.91 | container log: (EngineCore pid=169) INFO 10-15 05:13:00 [default_loader.py:430] Loading weights took 11.91 seconds | +| sampler.model_loaded | 05:13:01.578 | 76.4 | 13.84558 | container log: (EngineCore pid=169) INFO 10-15 05:13:01 [model_runner.py:428] Model loading took 18.17 GiB memory and 13.845580 seconds | +| sampler.compile_done | 05:13:38.517 | 113.4 | 35.91 | container log: (EngineCore pid=169) INFO 10-15 05:13:38 [monitor.py:53] torch.compile took 35.91 s in total | +| sampler.graphs_captured | 05:13:58.672 | 133.5 | 16.0 | container log: (EngineCore pid=169) INFO 10-15 05:13:58 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB | +| sampler.kv_cache_sized | 05:13:59.319 | 134.2 | | container log: (EngineCore pid=169) INFO 10-15 05:13:59 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB | +| sampler.engine_init_done | 05:14:17.795 | 152.6 | 76.21 | container log: (EngineCore pid=169) INFO 10-15 05:14:17 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.21 s | +| sampler.ready_key | 05:14:18.000 | 152.8 | | redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 05:11:45.156 | -0.0 | +| client.create_model.start | 05:11:45.157 | 0.0 | +| client.create_model.done | 05:12:10.447 | 25.3 | +| client.forward_backward.start | 05:12:13.323 | 28.2 | +| client.forward_backward.done | 05:12:16.232 | 31.1 | +| client.optim_step.done | 05:12:18.352 | 33.2 | +| client.save_weights_and_get_sampling_client.start | 05:12:18.353 | 33.2 | +| client.save_weights_and_get_sampling_client.done | 05:12:20.273 | 35.1 | +| client.sample.start | 05:12:20.275 | 35.1 | +| client.sample.done | 05:14:19.688 | 154.5 | +| client.end | 05:14:19.689 | 154.5 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.8 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 3.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.5 | +| weights load | trainer.load_start | trainer.load_done | 13.5 | +| ready key | trainer.load_done | trainer.ready_key | -27.0 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 25.3 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.4 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 15.4 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 9.5 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 12.9 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.6 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.3 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.2 | +| client: first sample blocked | client.sample.start | client.sample.done | 119.4 | + diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt new file mode 100644 index 00000000..b0d2d90a --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T05:11:40.798+00:00 diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt new file mode 100644 index 00000000..a9917932 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json new file mode 100644 index 00000000..467556c3 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:18Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041143235791008", + "uid": "996528ae-d593-4922-86b8-ea1303243615" + }, + "spec": { + "accelerator": { + "memory": "29Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "20f5c64d-1657-4c78-92ee-24fa609f498b", + "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:12:23Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-996528ae", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "29Gi", + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:45Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041110758783004", + "uid": "e9ddbd1c-f7f3-431b-abf2-f9f7cbdac473" + }, + "spec": { + "accelerator": { + "memory": "20Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "9b290f43-cc99-4ea2-b399-4c09027c74d4", + "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:11:50Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "20Gi", + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log new file mode 100644 index 00000000..3fc7c971 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log @@ -0,0 +1,35 @@ +watching; recorders [19286, 19287, 19289, 19291, 19293, 19288, 19290, 19292] write into runs/2026-10-15_cold_start/after-D2-8b-h100-cached +collected 2 worker log(s) into runs/2026-10-15_cold_start/after-D2-8b-h100-cached +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.4 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.1 | +| weights load | trainer.load_start | trainer.load_done | 12.1 | +| ready key | trainer.load_done | trainer.ready_key | -24.3 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 24.1 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.1 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 15.5 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 9.7 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 13.1 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.2 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.5 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.9 | +| client: first sample blocked | client.sample.start | client.sample.done | 119.9 | + diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log new file mode 100644 index 00000000..b4c54e58 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log @@ -0,0 +1,32 @@ +2026-10-15T05:15:36.893+00:00 2026-10-15T05:15:36.893863054Z INFO: 127.0.0.1:35824 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T05:15:37.116+00:00 2026-10-15T05:15:37.118777792Z INFO: 127.0.0.1:35826 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T05:15:37.193+00:00 2026-10-15T05:15:37.194477785Z INFO: 127.0.0.1:35826 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T05:15:37.636+00:00 2026-10-15T05:15:37.638416243Z [API_SERVER] enqueued op=create_model request_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:15:37.637+00:00 2026-10-15T05:15:37.639053003Z INFO: 127.0.0.1:35832 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T05:15:40.746+00:00 2026-10-15T05:15:40.746550012Z INFO: 10.32.3.1:39766 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:15:47.379+00:00 2026-10-15T05:15:47.379417038Z INFO: 127.0.0.1:54214 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:15:50.746+00:00 2026-10-15T05:15:50.746722382Z INFO: 10.32.3.1:56336 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:16:00.752+00:00 2026-10-15T05:16:00.747547185Z INFO: 10.32.3.1:37336 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:16:00.959+00:00 2026-10-15T05:16:00.949527284Z INFO: 127.0.0.1:33956 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T05:16:03.946+00:00 2026-10-15T05:16:03.946007674Z [API_SERVER] enqueued op=forward_backward request_id=f37e31ba-ce81-44ac-8856-bfda6aa3c7e1 model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:16:03.946+00:00 2026-10-15T05:16:03.946615325Z INFO: 127.0.0.1:33956 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T05:16:06.905+00:00 2026-10-15T05:16:06.905826356Z [API_SERVER] enqueued op=optim_step request_id=957669d4-dc27-4c81-837d-2a8ecf9bb2f4 model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:16:06.905+00:00 2026-10-15T05:16:06.907396223Z INFO: 127.0.0.1:33956 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T05:16:09.100+00:00 2026-10-15T05:16:09.049018149Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=3209fcb1-7f4e-446c-95c6-94a7e30b047f model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 +2026-10-15T05:16:09.100+00:00 2026-10-15T05:16:09.049557898Z INFO: 127.0.0.1:33956 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK +2026-10-15T05:16:10.931+00:00 2026-10-15T05:16:10.746565689Z INFO: 10.32.3.1:45574 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:16:11.296+00:00 2026-10-15T05:16:11.293332808Z INFO: 127.0.0.1:36994 - "POST /api/v1/asample HTTP/1.1" 200 OK +2026-10-15T05:16:20.765+00:00 2026-10-15T05:16:20.746229306Z INFO: 10.32.3.1:38344 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:16:30.748+00:00 2026-10-15T05:16:30.746892251Z INFO: 10.32.3.1:54454 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:16:40.748+00:00 2026-10-15T05:16:40.747033006Z INFO: 10.32.3.1:59170 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:16:50.747+00:00 2026-10-15T05:16:50.747504698Z INFO: 10.32.3.1:43604 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:16:57.561+00:00 2026-10-15T05:16:57.560755551Z INFO: 127.0.0.1:36756 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:17:00.747+00:00 2026-10-15T05:17:00.747501058Z INFO: 10.32.3.1:45120 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:17:10.747+00:00 2026-10-15T05:17:10.746700462Z INFO: 10.32.3.1:46882 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:17:20.747+00:00 2026-10-15T05:17:20.747102135Z INFO: 10.32.3.1:37964 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:17:30.751+00:00 2026-10-15T05:17:30.746961545Z INFO: 10.32.3.1:40486 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:17:40.748+00:00 2026-10-15T05:17:40.747075210Z INFO: 10.32.3.1:48770 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:17:47.748+00:00 2026-10-15T05:17:47.748538952Z INFO: 127.0.0.1:47530 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:17:50.747+00:00 2026-10-15T05:17:50.746643982Z INFO: 10.32.3.1:50180 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:18:00.747+00:00 2026-10-15T05:18:00.746678248Z INFO: 10.32.3.1:38072 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:18:10.746+00:00 2026-10-15T05:18:10.746445484Z INFO: 10.32.3.1:42070 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl new file mode 100644 index 00000000..dfbb3c97 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl @@ -0,0 +1,11 @@ +{"ts": "2026-10-15T05:15:36.633+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T05:15:36.633+00:00", "t": 0.001, "phase": "create_model.start"} +{"ts": "2026-10-15T05:16:00.765+00:00", "t": 24.133, "phase": "create_model.done", "model_id": "ce0e7790-7ff6-44f1-9632-06939e7b1053"} +{"ts": "2026-10-15T05:16:03.882+00:00", "t": 27.25, "phase": "forward_backward.start"} +{"ts": "2026-10-15T05:16:06.825+00:00", "t": 30.192, "phase": "forward_backward.done"} +{"ts": "2026-10-15T05:16:08.923+00:00", "t": 32.29, "phase": "optim_step.done"} +{"ts": "2026-10-15T05:16:08.924+00:00", "t": 32.291, "phase": "save_weights_and_get_sampling_client.start"} +{"ts": "2026-10-15T05:16:11.071+00:00", "t": 34.438, "phase": "save_weights_and_get_sampling_client.done"} +{"ts": "2026-10-15T05:16:11.076+00:00", "t": 34.443, "phase": "sample.start"} +{"ts": "2026-10-15T05:18:10.965+00:00", "t": 154.332, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} +{"ts": "2026-10-15T05:18:10.966+00:00", "t": 154.334, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log new file mode 100644 index 00000000..fedb02d3 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log @@ -0,0 +1,12 @@ +[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.001s create_model.start +[cold-start] 24.133s create_model.done {'model_id': 'ce0e7790-7ff6-44f1-9632-06939e7b1053'} +[cold-start] 27.250s forward_backward.start +[cold-start] 30.192s forward_backward.done +[cold-start] 32.290s optim_step.done +[cold-start] 32.291s save_weights_and_get_sampling_client.start +[cold-start] 34.438s save_weights_and_get_sampling_client.done +[cold-start] 34.443s sample.start +[cold-start] 154.332s sample.done {'text': ' 4\n\nQuestion: What is '} +[cold-start] 154.334s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt new file mode 100644 index 00000000..9458726e --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T05:18:11.482+00:00 diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json new file mode 100644 index 00000000..6a2fe366 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json @@ -0,0 +1,3069 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:10Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:10Z", + "name": "cold-start-fio.18de9a0ef295773d", + "namespace": "openrl-system", + "resourceVersion": "1792039090760383011", + "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5be63862", + "namespace": "openrl-system", + "resourceVersion": "1792039092515007000", + "uid": "355b59b9-3661-499d-8640-3f40c3c89375" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5ecfa51a", + "namespace": "openrl-system", + "resourceVersion": "1792039092561663021", + "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f61ddd1e4", + "namespace": "openrl-system", + "resourceVersion": "1792039092605935011", + "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:40:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:40:41Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:40:41Z", + "name": "cold-start-fio.18de9a32081cf908", + "namespace": "openrl-system", + "resourceVersion": "1792039241446287009", + "uid": "14a62e42-13ba-4526-a04d-a934702e1163" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:18Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:18Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:18Z", + "name": "cold-start-fio.18de9a3a9867ce9f", + "namespace": "openrl-system", + "resourceVersion": "1792039278208207001", + "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad2bcac91", + "namespace": "openrl-system", + "resourceVersion": "1792039279188751014", + "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad378cd74", + "namespace": "openrl-system", + "resourceVersion": "1792039279219679006", + "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad5aed221", + "namespace": "openrl-system", + "resourceVersion": "1792039279235135013", + "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:11Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:11Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:11Z", + "name": "cold-start-fio.18de9a70cd2c6a68", + "namespace": "openrl-system", + "resourceVersion": "1792039511048479001", + "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "resourceVersion": "1791941114537823021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Stopping container api-server", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", + "namespace": "openrl-system", + "resourceVersion": "1792039545418735013", + "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "namespace": "openrl-system", + "resourceVersion": "1792039545300351007", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", + "namespace": "openrl-system", + "resourceVersion": "1792039545387567001", + "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" + }, + "reason": "SuccessfulDelete", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546048591003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", + "namespace": "openrl-system", + "resourceVersion": "1792039546102351012", + "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:49Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:49Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", + "namespace": "openrl-system", + "resourceVersion": "1792039549712335019", + "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", + "namespace": "openrl-system", + "resourceVersion": "1792039558721215017", + "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", + "namespace": "openrl-system", + "resourceVersion": "1792039558783199013", + "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", + "namespace": "openrl-system", + "resourceVersion": "1792039558862303009", + "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "namespace": "openrl-system", + "resourceVersion": "1792039545940671018", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", + "namespace": "openrl-system", + "resourceVersion": "1792039546068975009", + "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" + }, + "reason": "SuccessfulCreate", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545232655018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78c9668cf7", + "namespace": "openrl-system", + "resourceVersion": "1792039545327119005", + "uid": "7b1433df-3524-478b-aba2-8979950e245e" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545464319018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78ef8b9e1b", + "namespace": "openrl-system", + "resourceVersion": "1792039545956079020", + "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:15Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875490623001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:15Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", + "namespace": "openrl-system", + "resourceVersion": "1792039875609279013", + "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:19Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:19Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", + "namespace": "openrl-system", + "resourceVersion": "1792039879126319008", + "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", + "namespace": "openrl-system", + "resourceVersion": "1792040061645247020", + "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", + "namespace": "openrl-system", + "resourceVersion": "1792040061849391002", + "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", + "namespace": "openrl-system", + "resourceVersion": "1792040062252079023", + "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", + "namespace": "openrl-system", + "resourceVersion": "1792040192780783021", + "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264403887001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", + "namespace": "openrl-system", + "resourceVersion": "1792040264444463021", + "uid": "a85006a2-68a6-4919-b148-08d58b546d07" + }, + "reason": "FailedScheduling", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Warning" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264434463001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", + "namespace": "openrl-system", + "resourceVersion": "1792040264884223002", + "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" + }, + "reason": "NotTriggerScaleUp", + "reportingComponent": "cluster-autoscaler", + "reportingInstance": "", + "source": { + "component": "cluster-autoscaler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264893039001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", + "namespace": "openrl-system", + "resourceVersion": "1792040293171567001", + "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", + "namespace": "openrl-system", + "resourceVersion": "1792040296672127014", + "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", + "namespace": "openrl-system", + "resourceVersion": "1792040296703007017", + "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", + "namespace": "openrl-system", + "resourceVersion": "1792040296889487014", + "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", + "namespace": "openrl-system", + "resourceVersion": "1792040427147711019", + "uid": "cc4da270-def2-456f-9374-372be8c717e4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:38Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498635999001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:38Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:38Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", + "namespace": "openrl-system", + "resourceVersion": "1792040498767471000", + "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", + "namespace": "openrl-system", + "resourceVersion": "1792040502270255017", + "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", + "namespace": "openrl-system", + "resourceVersion": "1792040502296943006", + "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", + "namespace": "openrl-system", + "resourceVersion": "1792040502475407023", + "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:09Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:09Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:09Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", + "namespace": "openrl-system", + "resourceVersion": "1792040649924575012", + "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:52Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672220015012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:52Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", + "namespace": "openrl-system", + "resourceVersion": "1792039672350591019", + "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:56Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:56Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:56Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", + "namespace": "openrl-system", + "resourceVersion": "1792039676166239015", + "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", + "namespace": "openrl-system", + "resourceVersion": "1792039844457167000", + "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", + "namespace": "openrl-system", + "resourceVersion": "1792039844632591021", + "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:45Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:45Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", + "namespace": "openrl-system", + "resourceVersion": "1792039845170959016", + "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", + "namespace": "openrl-system", + "resourceVersion": "1792040192981183013", + "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:23Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040242926223012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:23Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:23Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", + "namespace": "openrl-system", + "resourceVersion": "1792040243043743024", + "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", + "namespace": "openrl-system", + "resourceVersion": "1792040246545487023", + "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", + "namespace": "openrl-system", + "resourceVersion": "1792040246576895006", + "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", + "namespace": "openrl-system", + "resourceVersion": "1792040246742815002", + "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", + "namespace": "openrl-system", + "resourceVersion": "1792040427285631023", + "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:16Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476805407012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:16Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", + "namespace": "openrl-system", + "resourceVersion": "1792040476942623014", + "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", + "namespace": "openrl-system", + "resourceVersion": "1792040480747055005", + "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", + "namespace": "openrl-system", + "resourceVersion": "1792040480787967019", + "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", + "namespace": "openrl-system", + "resourceVersion": "1792040480963103007", + "uid": "d3158913-c79e-43d4-b468-d748641d49fe" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:10Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:10Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", + "namespace": "openrl-system", + "resourceVersion": "1792040650191935010", + "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", + "namespace": "openrl-system", + "resourceVersion": "1792038835732623016", + "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:58Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818213471023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:58Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:58Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", + "namespace": "openrl-system", + "resourceVersion": "1792040818411407005", + "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:07:01Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:07:01Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:07:01Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", + "namespace": "openrl-system", + "resourceVersion": "1792040821861695012", + "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", + "namespace": "openrl-system", + "resourceVersion": "1792040908347887024", + "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", + "namespace": "openrl-system", + "resourceVersion": "1792040908367439012", + "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", + "namespace": "openrl-system", + "resourceVersion": "1792040908418671004", + "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:10:57Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:10:57Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:10:57Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bd8cbffc810", + "namespace": "openrl-system", + "resourceVersion": "1792041057212047016", + "uid": "4a67fe5e-01b5-4b84-9453-b20887271ae4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:18Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138729647023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:18Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:18Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bebcffc9f4c", + "namespace": "openrl-system", + "resourceVersion": "1792041138871999014", + "uid": "3e19d3e2-4122-4f9d-8324-8f74961c36bc" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9c8a8459", + "namespace": "openrl-system", + "resourceVersion": "1792041142294159019", + "uid": "aaaf55ef-ec50-4fa5-b6a7-28a27d836f8e" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9d08793e", + "namespace": "openrl-system", + "resourceVersion": "1792041142311631019", + "uid": "5ab187bd-863a-4013-9e2d-185b71c68ead" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9beca1f20c38", + "namespace": "openrl-system", + "resourceVersion": "1792041142412303013", + "uid": "193a6a12-3e5a-44e3-8d69-ea9d46ee4707" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:14:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:14:49Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:14:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c0ec69a2183", + "namespace": "openrl-system", + "resourceVersion": "1792041289046591010", + "uid": "7161d68e-6e82-48a1-894f-7f149cadb6db" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:09Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369352463023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:09Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:09Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2181d762b3", + "namespace": "openrl-system", + "resourceVersion": "1792041369496943024", + "uid": "6a40425d-424d-438f-b398-45410b5817eb" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:13Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369459823023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:13Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:13Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c22606a262a", + "namespace": "openrl-system", + "resourceVersion": "1792041373214223010", + "uid": "c67940e2-268e-4a9a-94d4-05be2058728b" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:13Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369459823023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:13Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:13Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2260e74a63", + "namespace": "openrl-system", + "resourceVersion": "1792041373233151005", + "uid": "ecec41f0-1307-4892-ad43-57226155574a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:13Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369459823023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:13Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:13Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2266080366", + "namespace": "openrl-system", + "resourceVersion": "1792041373308543020", + "uid": "6dc932f6-ae1f-48f6-b85d-e087521c8e20" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", + "namespace": "openrl-system", + "resourceVersion": "1792038835415695012", + "uid": "918abc9a-1f24-4767-8928-b202482f11ed" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:59Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699253647006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:59Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:59Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", + "namespace": "openrl-system", + "resourceVersion": "1792040699411615020", + "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:05:02Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:05:02Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:05:02Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", + "namespace": "openrl-system", + "resourceVersion": "1792040702849391020", + "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", + "namespace": "openrl-system", + "resourceVersion": "1792040789347727015", + "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", + "namespace": "openrl-system", + "resourceVersion": "1792040789380863020", + "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", + "namespace": "openrl-system", + "resourceVersion": "1792040789417615006", + "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:10:57Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:10:57Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:10:57Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9bd8d38751ab", + "namespace": "openrl-system", + "resourceVersion": "1792041057319999002", + "uid": "0af06e74-e0c1-44de-a612-6fba47fb2b5c" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106258495006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:46Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:46Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be43ff3ded1", + "namespace": "openrl-system", + "resourceVersion": "1792041106381007003", + "uid": "4842f6c0-d606-444b-a970-6e9484189707" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50d27add6", + "namespace": "openrl-system", + "resourceVersion": "1792041109821407010", + "uid": "4e9749a9-e70a-453f-b51f-4d5ea7a60e86" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50daf2189", + "namespace": "openrl-system", + "resourceVersion": "1792041109840015005", + "uid": "2950a976-a558-471f-94fb-1ac680004e4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be5134afada", + "namespace": "openrl-system", + "resourceVersion": "1792041109931471020", + "uid": "d1c0b2ae-50a0-41e8-8333-441dd2250b23" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:14:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:14:49Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:14:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c0ece29f9b7", + "namespace": "openrl-system", + "resourceVersion": "1792041289153695008", + "uid": "9db24059-5412-4998-8f1e-ff4723a02422" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:37Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337881599006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:37Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:37Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1a2ddca97f", + "namespace": "openrl-system", + "resourceVersion": "1792041338007727010", + "uid": "622520b7-a0c6-4899-9d8e-8b7e289a5c19" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337986559006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:41Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:41Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afae88647", + "namespace": "openrl-system", + "resourceVersion": "1792041341445023014", + "uid": "174539ca-68ae-411a-9602-b94910a4f8a1" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337986559006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:41Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:41Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afb72b4d7", + "namespace": "openrl-system", + "resourceVersion": "1792041341467327017", + "uid": "abc887ed-fbc2-4086-9892-21b9dbd9ef25" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337986559006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:41Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:41Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1b0091bf79", + "namespace": "openrl-system", + "resourceVersion": "1792041341538399007", + "uid": "ce445484-5b88-4ca9-a93d-806ea29b103f" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt new file mode 100644 index 00000000..2a05dc10 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt @@ -0,0 +1,222 @@ +2026-10-15T05:16:28.490961146Z INFO 10-15 05:16:28 [model.py:692] Resolved architecture: Qwen3ForCausalLM +2026-10-15T05:16:28.491280812Z INFO 10-15 05:16:28 [model.py:2030] Using max model len 8192 +2026-10-15T05:16:28.492938929Z INFO 10-15 05:16:28 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. +2026-10-15T05:16:28.493837169Z INFO 10-15 05:16:28 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) +2026-10-15T05:16:31.024763096Z WARNING 10-15 05:16:31 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized +2026-10-15T05:16:38.151495536Z (EngineCore pid=169) INFO 10-15 05:16:38 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) +2026-10-15T05:16:38.483358659Z (EngineCore pid=169) INFO 10-15 05:16:38 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_4d2a060e6dcc492b933d56b85af7d4e7 backend=nccl +2026-10-15T05:16:38.487228505Z (EngineCore pid=169) INFO 10-15 05:16:38 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A +2026-10-15T05:16:38.487263057Z (EngineCore pid=169) INFO 10-15 05:16:38 [gpu_worker.py:441] Using V2 Model Runner +2026-10-15T05:16:39.244394278Z (EngineCore pid=169) INFO 10-15 05:16:39 [model_runner.py:396] Loading model from scratch... +2026-10-15T05:16:39.815914136Z (EngineCore pid=169) INFO 10-15 05:16:39 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +2026-10-15T05:16:39.816213542Z (EngineCore pid=169) INFO 10-15 05:16:39 [flash_attn.py:1116] Using FlashAttention version 3 +2026-10-15T05:16:39.931406255Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +2026-10-15T05:16:39.931446653Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +2026-10-15T05:16:40.237737879Z (EngineCore pid=169) INFO 10-15 05:16:40 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. +2026-10-15T05:16:40.237765322Z (EngineCore pid=169) INFO 10-15 05:16:40 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) +2026-10-15T05:16:40.238500999Z (EngineCore pid=169) +Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T05:15:45.904087714Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T05:15:45.904110382Z +2026-10-15T05:15:46.108440892Z Loading base model Qwen/Qwen3-8B to cuda:0... +2026-10-15T05:15:46.235847013Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T05:15:47.098453469Z [prefetch] reading 5 file(s), 15.26 GiB, from nfs into the page cache with 8 threads +2026-10-15T05:15:55.597395640Z [prefetch] done: 15.26 GiB in 8.5 s (1838 MiB/s) +2026-10-15T05:15:58.104601379Z +Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true +gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false +gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true +gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false +gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false +gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json new file mode 100644 index 00000000..2f350639 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json @@ -0,0 +1,6782 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:12:28Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-66wh7", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764209195247009", + "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-cwdlp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:16:49Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-cwdlp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:12:28Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-cm7x6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945508482719021", + "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-zbkzx", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:28Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:27Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-zbkzx", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:19:20Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-kpp7j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791945505996447017", + "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-v9zg5", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:38:25Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:38:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-v9zg5", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:36:04Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-rz7fw", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762451790735011", + "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-mrg4h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:31Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:47:31Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mrg4h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:36:04Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:47:32Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-t99wd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791762969945423016", + "uid": "24ddf14d-a173-4360-b179-064929dbff94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-j7fh2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:56:09Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:47:32Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:56:09Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-j7fh2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:47:32Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:03:54Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-accel-timeslicer-vlvfv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791763946154111000", + "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9c6fw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:12:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:03:54Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:12:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9c6fw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:03:54Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-12T00:16:49Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-vvwn6", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791764758862255017", + "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-rfs22", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:25:58Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-12T00:16:49Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-12T00:25:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-rfs22", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-12T00:16:49Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-11T23:26:48Z", + "generateName": "open-rl-accel-timeslicer-", + "generation": 1, + "labels": { + "app": "open-rl-accel-timeslicer", + "controller-revision-hash": "74748679dc", + "pod-template-generation": "3", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "open-rl-accel-timeslicer-xqx9j", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "open-rl-accel-timeslicer", + "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" + } + ], + "resourceVersion": "1791761764168719003", + "uid": "1b4d7707-110e-46db-84de-c07d21826f92" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--listen-host", + "0.0.0.0", + "--port", + "9753", + "--backend", + "llmd", + "--llmd-snapshot-endpoint", + "127.0.0.1:9001", + "--scheduling-policy", + "lrs" + ], + "command": [ + "uv", + "run", + "python", + "-m", + "accel_timeslicer.serve" + ], + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "name": "accel-timeslicer", + "ports": [ + { + "containerPort": 9753, + "hostPort": 9753, + "name": "snapshot", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirstWithHostNet", + "enableServiceLinks": true, + "hostNetwork": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 1000000, + "priorityClassName": "openrl-node-daemon", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "kube-api-access-9tztl", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:36:04Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-11T23:26:48Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "250m", + "memory": "256Mi" + }, + "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", + "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", + "lastState": {}, + "name": "accel-timeslicer", + "ready": true, + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "256Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-11T23:36:03Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-9tztl", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-11T23:26:48Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "generateName": "open-rl-api-server-6d86b498f5-", + "generation": 1, + "labels": { + "app": "open-rl-api-server", + "pod-template-hash": "6d86b498f5", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + } + ], + "resourceVersion": "1792039570790271003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "spec": { + "containers": [ + { + "command": [ + "uv", + "run", + "uvicorn", + "server.api_server:app", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "env": [ + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "true" + }, + { + "name": "ENABLE_GCP_TRACE", + "valueFrom": { + "configMapKeyRef": { + "key": "ENABLE_GCP_TRACE", + "name": "open-rl-config" + } + } + }, + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "BASE_MODEL", + "valueFrom": { + "configMapKeyRef": { + "key": "BASE_MODEL", + "name": "open-rl-config" + } + } + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "SAMPLING_BACKEND", + "value": "vllm" + }, + { + "name": "OPEN_RL_WORKER_MANAGER", + "value": "scheduler" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_WORKER_IMAGE", + "valueFrom": { + "configMapKeyRef": { + "key": "worker", + "name": "open-rl-images" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "api-server", + "ports": [ + { + "containerPort": 8000, + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/api/v1/healthz", + "port": 8000, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-2ns79", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:59Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:46:10Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T04:45:46Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "500m", + "memory": "1Gi" + }, + "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", + "lastState": {}, + "name": "api-server", + "ready": true, + "resources": { + "limits": { + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T04:45:58Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-2ns79", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.11", + "podIPs": [ + { + "ip": "10.32.3.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T04:45:46Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:25:14Z", + "generateName": "open-rl-scheduler-7cf9d4bc47-", + "generation": 1, + "labels": { + "app": "open-rl-scheduler", + "pod-template-hash": "7cf9d4bc47", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "open-rl-scheduler-7cf9d4bc47", + "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" + } + ], + "resourceVersion": "1791941144533087007", + "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" + }, + "spec": { + "containers": [ + { + "args": [ + "--leader-elect", + "--health-probe-bind-address=:8081", + "--metrics-bind-address=:8080" + ], + "env": [ + { + "name": "OPEN_RL_PLACEMENT_TIMEOUT", + "value": "0" + }, + { + "name": "OPEN_RL_WORKER_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "OPEN_RL_RECONCILE_INTERVAL", + "value": "10s" + }, + { + "name": "OPEN_RL_DEVICE_CLASS", + "value": "gpu.nvidia.com" + }, + { + "name": "OPEN_RL_DEVICE_DRIVER", + "value": "gpu.nvidia.com" + } + ], + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "manager", + "ports": [ + { + "containerPort": 8080, + "name": "metrics", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/readyz", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + }, + "readOnlyRootFilesystem": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "runAsNonRoot": true + }, + "serviceAccount": "open-rl-sa", + "serviceAccountName": "open-rl-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-kwbhw", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:44Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:25:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "100m", + "memory": "128Mi" + }, + "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", + "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", + "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", + "lastState": {}, + "name": "manager", + "ready": true, + "resources": { + "limits": { + "memory": "512Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:25:32Z" + } + }, + "user": { + "linux": { + "gid": 65532, + "supplementalGroups": [ + 65532 + ], + "uid": 65532 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-kwbhw", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.214", + "hostIPs": [ + { + "ip": "10.128.15.214" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.3.9", + "podIPs": [ + { + "ip": "10.32.3.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:25:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:09Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-sampler-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", + "openrl.io/role": "sampler", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-sampler", + "uid": "0c46d0b6-726c-4452-a321-7e87b66cd63b" + } + ], + "resourceVersion": "1792041373676703023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/sampler", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-sampler-0c46d0b6" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "35cbcea6-3b99-4526-8bc3-c6a94bb35da6" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-vpgx4", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-vpgx4", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:16:13Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:16:09Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:16:13Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:16:13Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:16:09Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "40Gi" + }, + "containerID": "containerd://ca45366dcf496a89394a485c3d9c32bb5a3e61633815714b2cbba49f4e1c81e6", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:16:13Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-vpgx4", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.1.11", + "podIPs": [ + { + "ip": "10.32.1.11" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:16:09Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:37Z", + "generation": 1, + "labels": { + "accel-timeslicer": "true", + "app": "open-rl-trainer-worker", + "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", + "openrl.io/role": "trainer", + "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", + "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", + "timeslice.io/owner": "qwen-qwen3-8b", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "openrl.io/v1alpha1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "Workload", + "name": "lora-qwen-qwen3-8b-0-trainer", + "uid": "d1f0b234-5386-41b2-b784-e32434f43d0b" + } + ], + "resourceVersion": "1792041342220143006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "In", + "values": [ + "true" + ] + } + ] + }, + { + "matchExpressions": [ + { + "key": "openrl.io/enabled", + "operator": "In", + "values": [ + "true" + ] + }, + { + "key": "openrl.io/trainer", + "operator": "DoesNotExist" + }, + { + "key": "openrl.io/sampler", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + }, + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.hostIP" + } + } + }, + { + "name": "OPEN_RL_TIME_SLICE_GROUP", + "value": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234" + }, + { + "name": "OPEN_RL_CLAIM_LEDGER", + "value": "ledger-lora-qwen-qwen3-8b-0-trainer-d1f0b234" + }, + { + "name": "OPEN_RL_ASSIGNMENT_ID", + "value": "ea5c40aa-ef66-4cb9-ada2-49e019e38b8f" + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imagePullPolicy": "IfNotPresent", + "name": "worker", + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-t494m", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "resourceClaims": [ + { + "name": "gpu", + "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234" + } + ], + "restartPolicy": "OnFailure", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + }, + { + "name": "kube-api-access-t494m", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:15:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:15:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:15:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:15:42Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-15T05:15:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "memory": "36Gi" + }, + "containerID": "containerd://cebaa8843ee5dc2b32cec11699295319c90e89cd67412af9420e375cdf94cfb7", + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", + "lastState": {}, + "name": "worker", + "ready": true, + "resources": { + "claims": [ + { + "name": "gpu" + } + ], + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-15T05:15:41Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-t494m", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.4.12", + "podIPs": [ + { + "ip": "10.32.4.12" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-15T05:15:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T01:28:42Z", + "generateName": "redis-store-86779df594-", + "generation": 1, + "labels": { + "app": "redis", + "pod-template-hash": "86779df594", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "redis-store-86779df594-7dc5x", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "redis-store-86779df594", + "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" + } + ], + "resourceVersion": "1791941333868287008", + "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" + }, + "spec": { + "containers": [ + { + "image": "redis:7-alpine", + "imagePullPolicy": "IfNotPresent", + "name": "redis", + "ports": [ + { + "containerPort": 6379, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-fxgk2", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:53Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T01:28:42Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "allocatedResources": { + "cpu": "1", + "memory": "4Gi" + }, + "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", + "image": "docker.io/library/redis:7-alpine", + "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", + "lastState": {}, + "name": "redis", + "ready": true, + "resources": { + "limits": { + "memory": "4Gi" + }, + "requests": { + "cpu": "1", + "memory": "4Gi" + } + }, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T01:28:52Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0, + 1, + 2, + 3, + 4, + 6, + 10, + 11, + 20, + 26, + 27 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-fxgk2", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.207", + "hostIPs": [ + { + "ip": "10.128.15.207" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.32.12.9", + "podIPs": [ + { + "ip": "10.32.12.9" + } + ], + "qosClass": "Burstable", + "startTime": "2026-10-14T01:28:42Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:05:36Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-9pkbv", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887950213247008", + "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-mm47f", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:50Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:05:37Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:05:50Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-mm47f", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.201", + "hostIPs": [ + { + "ip": "10.128.15.201" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.201", + "podIPs": [ + { + "ip": "10.128.15.201" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:05:37Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:18:38Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-9wd9k", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944324015471012", + "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-tuqu" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-5pb8r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:43Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:18:38Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:18:43Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-5pb8r", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.216", + "hostIPs": [ + { + "ip": "10.128.15.216" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.216", + "podIPs": [ + { + "ip": "10.128.15.216" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:18:38Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:22:33Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-b5pg4", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789888987151695021", + "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pgqxm", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:23:07Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:22:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:23:06Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pgqxm", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.202", + "hostIPs": [ + { + "ip": "10.128.15.202" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.202", + "podIPs": [ + { + "ip": "10.128.15.202" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:22:33Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:03:24Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-dclxl", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789887814636575001", + "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-tq64h", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:34Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:03:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:03:34Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-tq64h", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.196", + "hostIPs": [ + { + "ip": "10.128.15.196" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.196", + "podIPs": [ + { + "ip": "10.128.15.196" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:03:24Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-05T18:38:27Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-hcb9v", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791225513211215020", + "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-0853" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-pbcm9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:33Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-05T18:38:27Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-05T18:38:32Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-pbcm9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.197", + "hostIPs": [ + { + "ip": "10.128.15.197" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.197", + "podIPs": [ + { + "ip": "10.128.15.197" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-05T18:38:27Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-20T07:24:14Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-b" + }, + "name": "snapshot-agent-k8qtd", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1789889066585295015", + "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-bkd68", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:26Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-20T07:24:14Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-20T07:24:25Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-bkd68", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.194", + "hostIPs": [ + { + "ip": "10.128.15.194" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.194", + "podIPs": [ + { + "ip": "10.128.15.194" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-20T07:24:14Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-09-30T18:57:25Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-lgbqp", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1790794650142511011", + "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-sampler-a721f127-ed3t" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-966cp", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:30Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-09-30T18:57:25Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-09-30T18:57:29Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-966cp", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.0.10", + "hostIPs": [ + { + "ip": "10.128.0.10" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.0.10", + "podIPs": [ + { + "ip": "10.128.0.10" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-09-30T18:57:25Z" + } + }, + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2026-10-14T02:19:20Z", + "generateName": "snapshot-agent-", + "generation": 1, + "labels": { + "app.kubernetes.io/instance": "snapshot-agent", + "app.kubernetes.io/name": "snapshot-agent", + "controller-revision-hash": "687c7f8b9f", + "pod-template-generation": "1", + "topology.kubernetes.io/region": "us-central1", + "topology.kubernetes.io/zone": "us-central1-a" + }, + "name": "snapshot-agent-vt9ct", + "namespace": "openrl-system", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "snapshot-agent", + "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" + } + ], + "resourceVersion": "1791944364158111002", + "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" + }, + "spec": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchFields": [ + { + "key": "metadata.name", + "operator": "In", + "values": [ + "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + ] + } + ] + } + ] + } + } + }, + "containers": [ + { + "env": [ + { + "name": "DEPLOYMENT_MODE", + "value": "k8s" + }, + { + "name": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "AGENT_PORT", + "value": "9001" + }, + { + "name": "LD_LIBRARY_PATH", + "value": "/usr/local/nvidia/lib64" + }, + { + "name": "NVIDIA_VISIBLE_DEVICES", + "value": "all" + }, + { + "name": "NVIDIA_DRIVER_CAPABILITIES", + "value": "compute,utility" + } + ], + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imagePullPolicy": "Always", + "name": "snapshot-agent", + "ports": [ + { + "containerPort": 9001, + "hostPort": 9001, + "name": "grpc", + "protocol": "TCP" + } + ], + "resources": {}, + "securityContext": { + "privileged": true + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostNetwork": true, + "hostPID": true, + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "nodeSelector": { + "nvidia.com/gpu.present": "true" + }, + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "snapshot-agent", + "serviceAccountName": "snapshot-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/disk-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/pid-pressure", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/unschedulable", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node.kubernetes.io/network-unavailable", + "operator": "Exists" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/home/kubernetes/bin/nvidia", + "type": "Directory" + }, + "name": "nvidia-driver" + }, + { + "hostPath": { + "path": "/dev", + "type": "Directory" + }, + "name": "nvidia-devices" + }, + { + "name": "kube-api-access-6vrn7", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:24Z", + "observedGeneration": 1, + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2026-10-14T02:19:20Z", + "observedGeneration": 1, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", + "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", + "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", + "lastState": {}, + "name": "snapshot-agent", + "ready": true, + "resources": {}, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2026-10-14T02:19:23Z" + } + }, + "user": { + "linux": { + "gid": 0, + "supplementalGroups": [ + 0 + ], + "uid": 0 + } + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia-driver", + "readOnly": true, + "recursiveReadOnly": "Disabled" + }, + { + "mountPath": "/dev", + "name": "nvidia-devices" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-6vrn7", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "10.128.15.211", + "hostIPs": [ + { + "ip": "10.128.15.211" + } + ], + "observedGeneration": 1, + "phase": "Running", + "podIP": "10.128.15.211", + "podIPs": [ + { + "ip": "10.128.15.211" + } + ], + "qosClass": "BestEffort", + "startTime": "2026-10-14T02:19:20Z" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log new file mode 100644 index 00000000..5d1efc70 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log @@ -0,0 +1,334 @@ +2026-10-15T05:15:34.793+00:00 2026-10-15T05:15:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:34.794+00:00 2026-10-15T05:15:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:35.797+00:00 2026-10-15T05:15:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:35.797+00:00 2026-10-15T05:15:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:36.803+00:00 2026-10-15T05:15:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:36.803+00:00 2026-10-15T05:15:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:37.819+00:00 2026-10-15T05:15:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:37.819+00:00 2026-10-15T05:15:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:38.813+00:00 2026-10-15T05:15:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:38.814+00:00 2026-10-15T05:15:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:39.820+00:00 2026-10-15T05:15:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:39.820+00:00 2026-10-15T05:15:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:40.824+00:00 2026-10-15T05:15:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:40.824+00:00 2026-10-15T05:15:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:41.830+00:00 2026-10-15T05:15:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:41.830+00:00 2026-10-15T05:15:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:42.835+00:00 2026-10-15T05:15:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:42.835+00:00 2026-10-15T05:15:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:43.841+00:00 2026-10-15T05:15:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:43.841+00:00 2026-10-15T05:15:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:44.847+00:00 2026-10-15T05:15:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:44.847+00:00 2026-10-15T05:15:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:45.851+00:00 2026-10-15T05:15:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:45.851+00:00 2026-10-15T05:15:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:46.857+00:00 2026-10-15T05:15:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:46.857+00:00 2026-10-15T05:15:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:47.863+00:00 2026-10-15T05:15:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:47.863+00:00 2026-10-15T05:15:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:48.870+00:00 2026-10-15T05:15:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:48.870+00:00 2026-10-15T05:15:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:49.873+00:00 2026-10-15T05:15:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:49.873+00:00 2026-10-15T05:15:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:50.878+00:00 2026-10-15T05:15:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:50.878+00:00 2026-10-15T05:15:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:51.888+00:00 2026-10-15T05:15:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:51.889+00:00 2026-10-15T05:15:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:52.891+00:00 2026-10-15T05:15:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:52.892+00:00 2026-10-15T05:15:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:53.895+00:00 2026-10-15T05:15:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:53.895+00:00 2026-10-15T05:15:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:54.899+00:00 2026-10-15T05:15:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:54.899+00:00 2026-10-15T05:15:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:55.904+00:00 2026-10-15T05:15:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:55.904+00:00 2026-10-15T05:15:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:56.912+00:00 2026-10-15T05:15:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:56.912+00:00 2026-10-15T05:15:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:57.938+00:00 2026-10-15T05:15:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:57.938+00:00 2026-10-15T05:15:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:58.936+00:00 2026-10-15T05:15:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:58.936+00:00 2026-10-15T05:15:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:15:59.927+00:00 2026-10-15T05:15:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:15:59.927+00:00 2026-10-15T05:15:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:00.932+00:00 2026-10-15T05:16:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:00.932+00:00 2026-10-15T05:16:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:01.938+00:00 2026-10-15T05:16:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:01.938+00:00 2026-10-15T05:16:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:02.969+00:00 2026-10-15T05:16:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:02.970+00:00 2026-10-15T05:16:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:03.955+00:00 2026-10-15T05:16:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:03.955+00:00 2026-10-15T05:16:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:05.003+00:00 2026-10-15T05:16:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:05.004+00:00 2026-10-15T05:16:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:05.994+00:00 2026-10-15T05:16:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:05.995+00:00 2026-10-15T05:16:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:06.967+00:00 2026-10-15T05:16:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:06.967+00:00 2026-10-15T05:16:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:07.972+00:00 2026-10-15T05:16:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:07.972+00:00 2026-10-15T05:16:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:08.996+00:00 2026-10-15T05:16:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:08.996+00:00 2026-10-15T05:16:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:10.045+00:00 2026-10-15T05:16:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:10.045+00:00 2026-10-15T05:16:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:11.068+00:00 2026-10-15T05:16:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:11.068+00:00 2026-10-15T05:16:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:11.996+00:00 2026-10-15T05:16:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:11.997+00:00 2026-10-15T05:16:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:12.999+00:00 2026-10-15T05:16:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:12.999+00:00 2026-10-15T05:16:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:14.027+00:00 2026-10-15T05:16:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:14.028+00:00 2026-10-15T05:16:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:15.010+00:00 2026-10-15T05:16:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:15.011+00:00 2026-10-15T05:16:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:16.015+00:00 2026-10-15T05:16:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:16.015+00:00 2026-10-15T05:16:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:17.056+00:00 2026-10-15T05:16:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:17.057+00:00 2026-10-15T05:16:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:18.059+00:00 2026-10-15T05:16:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:18.059+00:00 2026-10-15T05:16:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:19.053+00:00 2026-10-15T05:16:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:19.053+00:00 2026-10-15T05:16:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:20.049+00:00 2026-10-15T05:16:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:20.049+00:00 2026-10-15T05:16:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:21.039+00:00 2026-10-15T05:16:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:21.039+00:00 2026-10-15T05:16:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:22.091+00:00 2026-10-15T05:16:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:22.091+00:00 2026-10-15T05:16:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:23.051+00:00 2026-10-15T05:16:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:23.051+00:00 2026-10-15T05:16:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:24.088+00:00 2026-10-15T05:16:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:24.088+00:00 2026-10-15T05:16:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:25.099+00:00 2026-10-15T05:16:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:25.099+00:00 2026-10-15T05:16:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:26.187+00:00 2026-10-15T05:16:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:26.187+00:00 2026-10-15T05:16:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:27.075+00:00 2026-10-15T05:16:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:27.075+00:00 2026-10-15T05:16:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:28.085+00:00 2026-10-15T05:16:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:28.086+00:00 2026-10-15T05:16:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:29.081+00:00 2026-10-15T05:16:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:29.081+00:00 2026-10-15T05:16:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:30.087+00:00 2026-10-15T05:16:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:30.087+00:00 2026-10-15T05:16:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:31.114+00:00 2026-10-15T05:16:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:31.114+00:00 2026-10-15T05:16:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:32.099+00:00 2026-10-15T05:16:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:32.099+00:00 2026-10-15T05:16:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:33.104+00:00 2026-10-15T05:16:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:33.104+00:00 2026-10-15T05:16:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:34.107+00:00 2026-10-15T05:16:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:34.108+00:00 2026-10-15T05:16:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:35.113+00:00 2026-10-15T05:16:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:35.113+00:00 2026-10-15T05:16:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:36.118+00:00 2026-10-15T05:16:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:36.119+00:00 2026-10-15T05:16:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:37.124+00:00 2026-10-15T05:16:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:37.125+00:00 2026-10-15T05:16:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:38.129+00:00 2026-10-15T05:16:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:38.129+00:00 2026-10-15T05:16:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:39.141+00:00 2026-10-15T05:16:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:39.141+00:00 2026-10-15T05:16:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:40.260+00:00 2026-10-15T05:16:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:40.260+00:00 2026-10-15T05:16:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:41.144+00:00 2026-10-15T05:16:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:41.144+00:00 2026-10-15T05:16:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:42.282+00:00 2026-10-15T05:16:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:42.282+00:00 2026-10-15T05:16:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:43.154+00:00 2026-10-15T05:16:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:43.154+00:00 2026-10-15T05:16:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:44.159+00:00 2026-10-15T05:16:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:44.159+00:00 2026-10-15T05:16:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:45.163+00:00 2026-10-15T05:16:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:45.163+00:00 2026-10-15T05:16:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:46.169+00:00 2026-10-15T05:16:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:46.169+00:00 2026-10-15T05:16:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:47.174+00:00 2026-10-15T05:16:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:47.175+00:00 2026-10-15T05:16:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:48.178+00:00 2026-10-15T05:16:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:48.178+00:00 2026-10-15T05:16:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:49.185+00:00 2026-10-15T05:16:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:49.185+00:00 2026-10-15T05:16:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:50.189+00:00 2026-10-15T05:16:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:50.189+00:00 2026-10-15T05:16:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:51.194+00:00 2026-10-15T05:16:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:51.195+00:00 2026-10-15T05:16:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:52.199+00:00 2026-10-15T05:16:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:52.199+00:00 2026-10-15T05:16:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:53.225+00:00 2026-10-15T05:16:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:53.225+00:00 2026-10-15T05:16:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:54.208+00:00 2026-10-15T05:16:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:54.208+00:00 2026-10-15T05:16:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:55.215+00:00 2026-10-15T05:16:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:55.215+00:00 2026-10-15T05:16:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:56.219+00:00 2026-10-15T05:16:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:56.219+00:00 2026-10-15T05:16:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:57.224+00:00 2026-10-15T05:16:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:57.224+00:00 2026-10-15T05:16:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:58.229+00:00 2026-10-15T05:16:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:58.229+00:00 2026-10-15T05:16:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:16:59.235+00:00 2026-10-15T05:16:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:16:59.235+00:00 2026-10-15T05:16:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:00.240+00:00 2026-10-15T05:17:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:00.240+00:00 2026-10-15T05:17:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:01.244+00:00 2026-10-15T05:17:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:01.245+00:00 2026-10-15T05:17:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:02.248+00:00 2026-10-15T05:17:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:02.249+00:00 2026-10-15T05:17:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:03.255+00:00 2026-10-15T05:17:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:03.255+00:00 2026-10-15T05:17:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:04.261+00:00 2026-10-15T05:17:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:04.261+00:00 2026-10-15T05:17:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:05.266+00:00 2026-10-15T05:17:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:05.266+00:00 2026-10-15T05:17:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:06.271+00:00 2026-10-15T05:17:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:06.272+00:00 2026-10-15T05:17:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:07.278+00:00 2026-10-15T05:17:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:07.278+00:00 2026-10-15T05:17:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:08.283+00:00 2026-10-15T05:17:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:08.283+00:00 2026-10-15T05:17:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:09.291+00:00 2026-10-15T05:17:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:09.291+00:00 2026-10-15T05:17:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:10.294+00:00 2026-10-15T05:17:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:10.294+00:00 2026-10-15T05:17:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:11.299+00:00 2026-10-15T05:17:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:11.299+00:00 2026-10-15T05:17:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:12.304+00:00 2026-10-15T05:17:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:12.304+00:00 2026-10-15T05:17:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:13.309+00:00 2026-10-15T05:17:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:13.309+00:00 2026-10-15T05:17:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:14.314+00:00 2026-10-15T05:17:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:14.314+00:00 2026-10-15T05:17:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:15.499+00:00 2026-10-15T05:17:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:15.499+00:00 2026-10-15T05:17:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:16.325+00:00 2026-10-15T05:17:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:16.325+00:00 2026-10-15T05:17:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:17.329+00:00 2026-10-15T05:17:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:17.330+00:00 2026-10-15T05:17:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:18.344+00:00 2026-10-15T05:17:18Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:18.344+00:00 2026-10-15T05:17:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:19.340+00:00 2026-10-15T05:17:19Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:19.340+00:00 2026-10-15T05:17:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:20.345+00:00 2026-10-15T05:17:20Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:20.345+00:00 2026-10-15T05:17:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:21.349+00:00 2026-10-15T05:17:21Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:21.349+00:00 2026-10-15T05:17:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:22.363+00:00 2026-10-15T05:17:22Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:22.364+00:00 2026-10-15T05:17:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:23.362+00:00 2026-10-15T05:17:23Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:23.362+00:00 2026-10-15T05:17:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:24.365+00:00 2026-10-15T05:17:24Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:24.365+00:00 2026-10-15T05:17:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:25.371+00:00 2026-10-15T05:17:25Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:25.371+00:00 2026-10-15T05:17:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:26.375+00:00 2026-10-15T05:17:26Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:26.375+00:00 2026-10-15T05:17:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:27.381+00:00 2026-10-15T05:17:27Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:27.381+00:00 2026-10-15T05:17:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:28.386+00:00 2026-10-15T05:17:28Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:28.386+00:00 2026-10-15T05:17:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:29.390+00:00 2026-10-15T05:17:29Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:29.391+00:00 2026-10-15T05:17:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:30.398+00:00 2026-10-15T05:17:30Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:30.398+00:00 2026-10-15T05:17:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:31.433+00:00 2026-10-15T05:17:31Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:31.434+00:00 2026-10-15T05:17:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:32.409+00:00 2026-10-15T05:17:32Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:32.409+00:00 2026-10-15T05:17:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:33.414+00:00 2026-10-15T05:17:33Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:33.414+00:00 2026-10-15T05:17:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:34.418+00:00 2026-10-15T05:17:34Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:34.419+00:00 2026-10-15T05:17:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:35.424+00:00 2026-10-15T05:17:35Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:35.424+00:00 2026-10-15T05:17:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:36.429+00:00 2026-10-15T05:17:36Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:36.429+00:00 2026-10-15T05:17:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:37.435+00:00 2026-10-15T05:17:37Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:37.436+00:00 2026-10-15T05:17:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:38.463+00:00 2026-10-15T05:17:38Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:38.463+00:00 2026-10-15T05:17:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:39.445+00:00 2026-10-15T05:17:39Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:39.445+00:00 2026-10-15T05:17:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:40.449+00:00 2026-10-15T05:17:40Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:40.449+00:00 2026-10-15T05:17:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:41.454+00:00 2026-10-15T05:17:41Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:41.454+00:00 2026-10-15T05:17:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:42.469+00:00 2026-10-15T05:17:42Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:42.469+00:00 2026-10-15T05:17:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:43.464+00:00 2026-10-15T05:17:43Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:43.465+00:00 2026-10-15T05:17:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:44.469+00:00 2026-10-15T05:17:44Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:44.469+00:00 2026-10-15T05:17:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:45.475+00:00 2026-10-15T05:17:45Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:45.475+00:00 2026-10-15T05:17:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:46.632+00:00 2026-10-15T05:17:46Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:46.632+00:00 2026-10-15T05:17:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:47.485+00:00 2026-10-15T05:17:47Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:47.485+00:00 2026-10-15T05:17:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:48.489+00:00 2026-10-15T05:17:48Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:48.490+00:00 2026-10-15T05:17:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:49.495+00:00 2026-10-15T05:17:49Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:49.495+00:00 2026-10-15T05:17:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:50.501+00:00 2026-10-15T05:17:50Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:50.501+00:00 2026-10-15T05:17:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:51.508+00:00 2026-10-15T05:17:51Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:51.508+00:00 2026-10-15T05:17:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:52.511+00:00 2026-10-15T05:17:52Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:52.511+00:00 2026-10-15T05:17:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:53.516+00:00 2026-10-15T05:17:53Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:53.517+00:00 2026-10-15T05:17:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:54.521+00:00 2026-10-15T05:17:54Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:54.521+00:00 2026-10-15T05:17:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:55.527+00:00 2026-10-15T05:17:55Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:55.527+00:00 2026-10-15T05:17:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:56.530+00:00 2026-10-15T05:17:56Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:56.530+00:00 2026-10-15T05:17:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:57.538+00:00 2026-10-15T05:17:57Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:57.538+00:00 2026-10-15T05:17:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:58.541+00:00 2026-10-15T05:17:58Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:58.541+00:00 2026-10-15T05:17:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:17:59.546+00:00 2026-10-15T05:17:59Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:17:59.546+00:00 2026-10-15T05:17:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:00.551+00:00 2026-10-15T05:18:00Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:00.552+00:00 2026-10-15T05:18:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:01.567+00:00 2026-10-15T05:18:01Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:01.567+00:00 2026-10-15T05:18:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:02.561+00:00 2026-10-15T05:18:02Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:02.561+00:00 2026-10-15T05:18:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:03.566+00:00 2026-10-15T05:18:03Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:03.566+00:00 2026-10-15T05:18:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:04.573+00:00 2026-10-15T05:18:04Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:04.573+00:00 2026-10-15T05:18:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:05.577+00:00 2026-10-15T05:18:05Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:05.577+00:00 2026-10-15T05:18:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:06.582+00:00 2026-10-15T05:18:06Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:06.582+00:00 2026-10-15T05:18:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:07.589+00:00 2026-10-15T05:18:07Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:07.589+00:00 2026-10-15T05:18:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:08.594+00:00 2026-10-15T05:18:08Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:08.595+00:00 2026-10-15T05:18:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:09.599+00:00 2026-10-15T05:18:09Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:09.599+00:00 2026-10-15T05:18:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:10.606+00:00 2026-10-15T05:18:10Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:10.606+00:00 2026-10-15T05:18:10Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:10.606+00:00 2026-10-15T05:18:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:11.610+00:00 2026-10-15T05:18:11Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:11.610+00:00 2026-10-15T05:18:11Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:11.610+00:00 2026-10-15T05:18:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:12.616+00:00 2026-10-15T05:18:12Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:12.616+00:00 2026-10-15T05:18:12Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:12.616+00:00 2026-10-15T05:18:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:13.638+00:00 2026-10-15T05:18:13Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:13.638+00:00 2026-10-15T05:18:13Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:13.638+00:00 2026-10-15T05:18:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:14.627+00:00 2026-10-15T05:18:14Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:14.628+00:00 2026-10-15T05:18:14Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:14.628+00:00 2026-10-15T05:18:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:15.631+00:00 2026-10-15T05:18:15Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:15.631+00:00 2026-10-15T05:18:15Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:15.631+00:00 2026-10-15T05:18:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:16.637+00:00 2026-10-15T05:18:16Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:16.637+00:00 2026-10-15T05:18:16Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:16.637+00:00 2026-10-15T05:18:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B +2026-10-15T05:18:17.642+00:00 2026-10-15T05:18:17Z open_rl:sampler_ready:Qwen/Qwen3-8B +2026-10-15T05:18:17.642+00:00 2026-10-15T05:18:17Z open_rl:trainer_ready:Qwen/Qwen3-8B +2026-10-15T05:18:17.642+00:00 2026-10-15T05:18:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log new file mode 100644 index 00000000..67f925c9 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log @@ -0,0 +1,2 @@ +2026-10-15T05:15:37.772+00:00 2026-10-15T05:15:37.770103723Z {"level":"info","ts":"2026-10-15T05:15:37Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"6e1c65ba-9430-4398-89db-8b05b9f2092e","claim":"claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} +2026-10-15T05:16:09.217+00:00 2026-10-15T05:16:09.211255887Z {"level":"info","ts":"2026-10-15T05:16:09Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"d4df84da-7e1d-497e-abe9-39f53b544303","claim":"claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json new file mode 100644 index 00000000..4035e7bf --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json @@ -0,0 +1,267 @@ +{ + "case": "after-D2-8b-h100-cached", + "collected": "2026-10-15T05:18:18.329+00:00", + "client": { + "client.start": { + "ts": "2026-10-15 05:15:36.633000+00:00", + "source": "client.jsonl (laptop clock)", + "base_model": "Qwen/Qwen3-8B", + "base_url": "http://127.0.0.1:9003" + }, + "client.create_model.start": { + "ts": "2026-10-15 05:15:36.633000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.create_model.done": { + "ts": "2026-10-15 05:16:00.765000+00:00", + "source": "client.jsonl (laptop clock)", + "model_id": "ce0e7790-7ff6-44f1-9632-06939e7b1053" + }, + "client.forward_backward.start": { + "ts": "2026-10-15 05:16:03.882000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.forward_backward.done": { + "ts": "2026-10-15 05:16:06.825000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.optim_step.done": { + "ts": "2026-10-15 05:16:08.923000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.start": { + "ts": "2026-10-15 05:16:08.924000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.save_weights_and_get_sampling_client.done": { + "ts": "2026-10-15 05:16:11.071000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.start": { + "ts": "2026-10-15 05:16:11.076000+00:00", + "source": "client.jsonl (laptop clock)" + }, + "client.sample.done": { + "ts": "2026-10-15 05:18:10.965000+00:00", + "source": "client.jsonl (laptop clock)", + "text": " 4\n\nQuestion: What is " + }, + "client.end": { + "ts": "2026-10-15 05:18:10.966000+00:00", + "source": "client.jsonl (laptop clock)" + } + }, + "workers": { + "sampler": { + "workload": "lora-qwen-qwen3-8b-0-sampler", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "pod": "orw-lora-qwen-qwen3-8b-0-sampler", + "marks": { + "sampler.workload_created": { + "ts": "2026-10-15 05:16:09+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "sampler.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:16:09.599000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "sampler.workload_reports_pod": { + "ts": "2026-10-15 05:16:09.599000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "sampler.workload_reports_node": { + "ts": "2026-10-15 05:16:09.599000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "sampler.condition_Placed_True": { + "ts": "2026-10-15 05:16:13+00:00", + "source": "Workload condition Placed" + }, + "sampler.pod_created": { + "ts": "2026-10-15 05:16:09+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "sampler.pod_Initialized": { + "ts": "2026-10-15 05:16:09+00:00", + "source": "pod condition Initialized" + }, + "sampler.pod_Ready": { + "ts": "2026-10-15 05:16:13+00:00", + "source": "pod condition Ready" + }, + "sampler.pod_ContainersReady": { + "ts": "2026-10-15 05:16:13+00:00", + "source": "pod condition ContainersReady" + }, + "sampler.pod_PodScheduled": { + "ts": "2026-10-15 05:16:09+00:00", + "source": "pod condition PodScheduled" + }, + "sampler.container_started": { + "ts": "2026-10-15 05:16:13+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "sampler.event_Scheduled": { + "ts": "2026-10-15 05:16:09+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "sampler.event_Pulled": { + "ts": "2026-10-15 05:16:13+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "sampler.event_Created": { + "ts": "2026-10-15 05:16:13+00:00", + "source": "event: Container created" + }, + "sampler.event_Started": { + "ts": "2026-10-15 05:16:13+00:00", + "source": "event: Container started" + }, + "sampler.process_first_log": { + "ts": "2026-10-15 05:16:28.490961+00:00", + "source": "container log, first line" + }, + "sampler.engine_init_start": { + "ts": "2026-10-15 05:16:38.151495+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:38 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" + }, + "sampler.load_start": { + "ts": "2026-10-15 05:16:39.244394+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:39 [model_runner.py:396] Loading model from scratch..." + }, + "sampler.prefetch_start": { + "ts": "2026-10-15 05:16:40.237765+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:40 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " + }, + "sampler.weights_loaded": { + "ts": "2026-10-15 05:16:52.391122+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:52 [default_loader.py:430] Loading weights took 12.16 seconds", + "self_reported_s": 12.16 + }, + "sampler.model_loaded": { + "ts": "2026-10-15 05:16:53.020747+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:53 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.053276 seconds", + "self_reported_s": 14.053276 + }, + "sampler.compile_done": { + "ts": "2026-10-15 05:17:29.578400+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:17:29 [monitor.py:53] torch.compile took 35.52 s in total", + "self_reported_s": 35.52 + }, + "sampler.graphs_captured": { + "ts": "2026-10-15 05:17:49.866634+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:17:49 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", + "self_reported_s": 16.0 + }, + "sampler.kv_cache_sized": { + "ts": "2026-10-15 05:17:50.489301+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:17:50 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" + }, + "sampler.engine_init_done": { + "ts": "2026-10-15 05:18:09.053659+00:00", + "source": "container log: (EngineCore pid=169) INFO 10-15 05:18:09 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.03 s", + "self_reported_s": 76.03 + }, + "sampler.ready_key": { + "ts": "2026-10-15 05:18:10+00:00", + "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + }, + "trainer": { + "workload": "lora-qwen-qwen3-8b-0-trainer", + "model_id": "Qwen/Qwen3-8B", + "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "pod": "orw-lora-qwen-qwen3-8b-0-trainer", + "marks": { + "trainer.workload_created": { + "ts": "2026-10-15 05:15:37+00:00", + "source": "Workload creationTimestamp (apiserver)" + }, + "trainer.workload_first_seen_by_poller": { + "ts": "2026-10-15 05:15:37.332000+00:00", + "source": "workloads.tsv poller (laptop clock, 1 s)" + }, + "trainer.workload_reports_pod": { + "ts": "2026-10-15 05:15:37.332000+00:00", + "source": "workloads.tsv poller: status.podName set" + }, + "trainer.workload_reports_node": { + "ts": "2026-10-15 05:15:38.981000+00:00", + "source": "workloads.tsv poller: status.nodeName set" + }, + "trainer.condition_Placed_True": { + "ts": "2026-10-15 05:15:42+00:00", + "source": "Workload condition Placed" + }, + "trainer.pod_created": { + "ts": "2026-10-15 05:15:37+00:00", + "source": "pod creationTimestamp (apiserver)" + }, + "trainer.pod_Initialized": { + "ts": "2026-10-15 05:15:38+00:00", + "source": "pod condition Initialized" + }, + "trainer.pod_Ready": { + "ts": "2026-10-15 05:15:42+00:00", + "source": "pod condition Ready" + }, + "trainer.pod_ContainersReady": { + "ts": "2026-10-15 05:15:42+00:00", + "source": "pod condition ContainersReady" + }, + "trainer.pod_PodScheduled": { + "ts": "2026-10-15 05:15:37+00:00", + "source": "pod condition PodScheduled" + }, + "trainer.container_started": { + "ts": "2026-10-15 05:15:41+00:00", + "source": "containerStatuses.state.running.startedAt" + }, + "trainer.event_Scheduled": { + "ts": "2026-10-15 05:15:37+00:00", + "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "trainer.event_Pulled": { + "ts": "2026-10-15 05:15:41+00:00", + "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", + "kubelet_pull_s": null, + "pulled_from_cache": true + }, + "trainer.event_Created": { + "ts": "2026-10-15 05:15:41+00:00", + "source": "event: Container created" + }, + "trainer.event_Started": { + "ts": "2026-10-15 05:15:41+00:00", + "source": "event: Container started" + }, + "trainer.process_first_log": { + "ts": "2026-10-15 05:15:45.886784+00:00", + "source": "container log, first line" + }, + "trainer.load_start": { + "ts": "2026-10-15 05:15:46.108440+00:00", + "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." + }, + "trainer.load_done": { + "ts": "2026-10-15 05:15:58.258108+00:00", + "source": "container log: Successfully loaded." + }, + "trainer.processor_started": { + "ts": "2026-10-15 05:15:58.259517+00:00", + "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." + }, + "trainer.ready_key": { + "ts": "2026-10-15 05:15:34+00:00", + "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" + } + }, + "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" + } + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.md b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.md new file mode 100644 index 00000000..9849c4aa --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.md @@ -0,0 +1,109 @@ +# Cold start timeline: after-D2-8b-h100-cached + +t=0 is the client's `create_model` call at 2026-10-15T05:15:36.633+00:00 UTC. Offsets are seconds after t=0; a negative offset is before it. + +## trainer: `orw-lora-qwen-qwen3-8b-0-trainer` on `gke-open-rl-dra-h100-trainer-702095b4-1g3a` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| trainer.ready_key | 05:15:34.000 | -2.6 | | redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B | +| trainer.workload_created | 05:15:37.000 | 0.4 | | Workload creationTimestamp (apiserver) | +| trainer.pod_created | 05:15:37.000 | 0.4 | | pod creationTimestamp (apiserver) | +| trainer.pod_PodScheduled | 05:15:37.000 | 0.4 | | pod condition PodScheduled | +| trainer.event_Scheduled | 05:15:37.000 | 0.4 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a | +| trainer.workload_first_seen_by_poller | 05:15:37.332 | 0.7 | | workloads.tsv poller (laptop clock, 1 s) | +| trainer.workload_reports_pod | 05:15:37.332 | 0.7 | | workloads.tsv poller: status.podName set | +| trainer.pod_Initialized | 05:15:38.000 | 1.4 | | pod condition Initialized | +| trainer.workload_reports_node | 05:15:38.981 | 2.3 | | workloads.tsv poller: status.nodeName set | +| trainer.container_started | 05:15:41.000 | 4.4 | | containerStatuses.state.running.startedAt | +| trainer.event_Pulled | 05:15:41.000 | 4.4 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| trainer.event_Created | 05:15:41.000 | 4.4 | | event: Container created | +| trainer.event_Started | 05:15:41.000 | 4.4 | | event: Container started | +| trainer.condition_Placed_True | 05:15:42.000 | 5.4 | | Workload condition Placed | +| trainer.pod_Ready | 05:15:42.000 | 5.4 | | pod condition Ready | +| trainer.pod_ContainersReady | 05:15:42.000 | 5.4 | | pod condition ContainersReady | +| trainer.process_first_log | 05:15:45.886 | 9.3 | | container log, first line | +| trainer.load_start | 05:15:46.108 | 9.5 | | container log: Loading base model Qwen/Qwen3-8B to cuda:0... | +| trainer.load_done | 05:15:58.258 | 21.6 | | container log: Successfully loaded. | +| trainer.processor_started | 05:15:58.259 | 21.6 | | container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1). | + +## sampler: `orw-lora-qwen-qwen3-8b-0-sampler` on `gke-open-rl-dra-h100-sampler-a721f127-0853` + +| Mark | Clock (UTC) | t+ s | Self-reported s | Source | +|---|---|---|---|---| +| sampler.workload_created | 05:16:09.000 | 32.4 | | Workload creationTimestamp (apiserver) | +| sampler.pod_created | 05:16:09.000 | 32.4 | | pod creationTimestamp (apiserver) | +| sampler.pod_Initialized | 05:16:09.000 | 32.4 | | pod condition Initialized | +| sampler.pod_PodScheduled | 05:16:09.000 | 32.4 | | pod condition PodScheduled | +| sampler.event_Scheduled | 05:16:09.000 | 32.4 | | event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853 | +| sampler.workload_first_seen_by_poller | 05:16:09.599 | 33.0 | | workloads.tsv poller (laptop clock, 1 s) | +| sampler.workload_reports_pod | 05:16:09.599 | 33.0 | | workloads.tsv poller: status.podName set | +| sampler.workload_reports_node | 05:16:09.599 | 33.0 | | workloads.tsv poller: status.nodeName set | +| sampler.condition_Placed_True | 05:16:13.000 | 36.4 | | Workload condition Placed | +| sampler.pod_Ready | 05:16:13.000 | 36.4 | | pod condition Ready | +| sampler.pod_ContainersReady | 05:16:13.000 | 36.4 | | pod condition ContainersReady | +| sampler.container_started | 05:16:13.000 | 36.4 | | containerStatuses.state.running.startedAt | +| sampler.event_Pulled | 05:16:13.000 | 36.4 | None | event: Container image "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a" already present on machine and | +| sampler.event_Created | 05:16:13.000 | 36.4 | | event: Container created | +| sampler.event_Started | 05:16:13.000 | 36.4 | | event: Container started | +| sampler.process_first_log | 05:16:28.490 | 51.9 | | container log, first line | +| sampler.engine_init_start | 05:16:38.151 | 61.5 | | container log: (EngineCore pid=169) INFO 10-15 05:16:38 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q | +| sampler.load_start | 05:16:39.244 | 62.6 | | container log: (EngineCore pid=169) INFO 10-15 05:16:39 [model_runner.py:396] Loading model from scratch... | +| sampler.prefetch_start | 05:16:40.237 | 63.6 | | container log: (EngineCore pid=169) INFO 10-15 05:16:40 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in | +| sampler.weights_loaded | 05:16:52.391 | 75.8 | 12.16 | container log: (EngineCore pid=169) INFO 10-15 05:16:52 [default_loader.py:430] Loading weights took 12.16 seconds | +| sampler.model_loaded | 05:16:53.020 | 76.4 | 14.053276 | container log: (EngineCore pid=169) INFO 10-15 05:16:53 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.053276 seconds | +| sampler.compile_done | 05:17:29.578 | 112.9 | 35.52 | container log: (EngineCore pid=169) INFO 10-15 05:17:29 [monitor.py:53] torch.compile took 35.52 s in total | +| sampler.graphs_captured | 05:17:49.866 | 133.2 | 16.0 | container log: (EngineCore pid=169) INFO 10-15 05:17:49 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB | +| sampler.kv_cache_sized | 05:17:50.489 | 133.9 | | container log: (EngineCore pid=169) INFO 10-15 05:17:50 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB | +| sampler.engine_init_done | 05:18:09.053 | 152.4 | 76.03 | container log: (EngineCore pid=169) INFO 10-15 05:18:09 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.03 s | +| sampler.ready_key | 05:18:10.000 | 153.4 | | redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B | + +## client + +| Phase | Clock (UTC) | t+ s | +|---|---|---| +| client.start | 05:15:36.633 | 0.0 | +| client.create_model.start | 05:15:36.633 | 0.0 | +| client.create_model.done | 05:16:00.765 | 24.1 | +| client.forward_backward.start | 05:16:03.882 | 27.2 | +| client.forward_backward.done | 05:16:06.825 | 30.2 | +| client.optim_step.done | 05:16:08.923 | 32.3 | +| client.save_weights_and_get_sampling_client.start | 05:16:08.924 | 32.3 | +| client.save_weights_and_get_sampling_client.done | 05:16:11.071 | 34.4 | +| client.sample.start | 05:16:11.076 | 34.4 | +| client.sample.done | 05:18:10.965 | 154.3 | +| client.end | 05:18:10.966 | 154.3 | + +## Critical path (derived from the marks above) + +### trainer + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.4 | +| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | +| container create | trainer.event_Pulled | trainer.container_started | 0.0 | +| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.1 | +| weights load | trainer.load_start | trainer.load_done | 12.1 | +| ready key | trainer.load_done | trainer.ready_key | -24.3 | +| client: create_model blocked | client.create_model.start | client.create_model.done | 24.1 | +| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | + +### sampler + +| Phase | From | To | Seconds | +|---|---|---|---| +| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.1 | +| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | +| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | +| container create | sampler.event_Pulled | sampler.container_started | 0.0 | +| process start to first log | sampler.container_started | sampler.process_first_log | 15.5 | +| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 9.7 | +| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | +| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 13.1 | +| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.2 | +| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.5 | +| ready key | sampler.engine_init_done | sampler.ready_key | 0.9 | +| client: first sample blocked | client.sample.start | client.sample.done | 119.9 | + diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt new file mode 100644 index 00000000..e83592ef --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt @@ -0,0 +1 @@ +2026-10-15T05:15:32.369+00:00 diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt new file mode 100644 index 00000000..a9917932 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt @@ -0,0 +1 @@ +us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json new file mode 100644 index 00000000..9df2e608 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json @@ -0,0 +1,355 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:09Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041373802335008", + "uid": "0c46d0b6-726c-4452-a321-7e87b66cd63b" + }, + "spec": { + "accelerator": { + "memory": "29Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "sampler", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.vllm_sampler" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "30725539152" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "OPEN_RL_MODEL_ID", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "VLLM_SERVER_DEV_MODE", + "value": "1" + }, + { + "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", + "value": "1" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-sampler" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "40Gi" + }, + "requests": { + "memory": "40Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "35cbcea6-3b99-4526-8bc3-c6a94bb35da6", + "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:16:13Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "29Gi", + "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-sampler" + } + }, + { + "apiVersion": "openrl.io/v1alpha1", + "kind": "Workload", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:37Z", + "finalizers": [ + "openrl.io/placement" + ], + "generation": 2, + "labels": { + "app.kubernetes.io/managed-by": "open-rl-api-server" + }, + "name": "lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041342325231004", + "uid": "d1f0b234-5386-41b2-b784-e32434f43d0b" + }, + "spec": { + "accelerator": { + "memory": "20Gi", + "mode": "SingleGPU" + }, + "exclusive": true, + "modelID": "Qwen/Qwen3-8B", + "ownerID": "qwen-qwen3-8b", + "role": "trainer", + "template": { + "metadata": {}, + "spec": { + "containers": [ + { + "args": [ + "--model-id", + "Qwen/Qwen3-8B", + "--active-tenant-set-id", + "Qwen/Qwen3-8B-1" + ], + "command": [ + "uv", + "run", + "python", + "-u", + "-m", + "server.training_requests_processor" + ], + "env": [ + { + "name": "REDIS_URL", + "value": "redis://redis-service:6379?max_connections=500" + }, + { + "name": "OPEN_RL_TMP_DIR", + "value": "/mnt/shared/open-rl" + }, + { + "name": "HF_HOME", + "value": "/mnt/shared/open-rl/huggingface" + }, + { + "name": "BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_BASE_MODEL", + "value": "Qwen/Qwen3-8B" + }, + { + "name": "OPEN_RL_ENABLE_FFT", + "value": "false" + }, + { + "name": "OPEN_RL_FINE_TUNING_TYPE", + "value": "lora" + }, + { + "name": "OPEN_RL_ACCELERATOR_MEMORY", + "value": "20674967296" + }, + { + "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", + "value": "delta" + }, + { + "name": "PYTORCH_CUDA_ALLOC_CONF", + "value": "expandable_segments:True" + }, + { + "name": "OPEN_RL_WORKLOAD_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_TIME_SLICE_JOB_ID", + "value": "lora-qwen-qwen3-8b-0-trainer" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", + "value": "9753" + }, + { + "name": "OPEN_RL_TIME_SLICING", + "value": "off" + }, + { + "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", + "valueFrom": { + "fieldRef": { + "fieldPath": "status.hostIP" + } + } + } + ], + "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", + "name": "worker", + "resources": { + "limits": { + "memory": "36Gi" + }, + "requests": { + "memory": "36Gi" + } + }, + "volumeMounts": [ + { + "mountPath": "/mnt/shared", + "name": "shared-storage" + } + ] + } + ], + "restartPolicy": "OnFailure", + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "operator": "Exists" + } + ], + "volumes": [ + { + "name": "shared-storage", + "persistentVolumeClaim": { + "claimName": "open-rl-shared-pvc" + } + } + ] + } + }, + "trainingKind": "lora", + "workerContainerName": "worker" + }, + "status": { + "assignmentID": "ea5c40aa-ef66-4cb9-ada2-49e019e38b8f", + "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", + "conditions": [ + { + "lastTransitionTime": "2026-10-15T05:15:42Z", + "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", + "reason": "Placed", + "status": "True", + "type": "Placed" + } + ], + "deviceCount": 1, + "memoryPerDevice": "20Gi", + "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "observedGeneration": 2, + "phase": "Running", + "podName": "orw-lora-qwen-qwen3-8b-0-trainer" + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log new file mode 100644 index 00000000..fc418348 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log @@ -0,0 +1,26 @@ +2026-10-15T05:19:17.204+00:00 2026-10-15T05:19:17.207002187Z INFO: 127.0.0.1:58394 - "POST /api/v1/client/config HTTP/1.1" 200 OK +2026-10-15T05:19:17.439+00:00 2026-10-15T05:19:17.441717541Z INFO: 127.0.0.1:58408 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found +2026-10-15T05:19:17.507+00:00 2026-10-15T05:19:17.509980459Z INFO: 127.0.0.1:58408 - "POST /api/v1/create_session HTTP/1.1" 200 OK +2026-10-15T05:19:17.741+00:00 2026-10-15T05:19:17.744372268Z [API_SERVER] enqueued op=create_model request_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 model_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 active_set=google/gemma-4-e2b-1 +2026-10-15T05:19:17.742+00:00 2026-10-15T05:19:17.745324931Z INFO: 127.0.0.1:58412 - "POST /api/v1/create_model HTTP/1.1" 200 OK +2026-10-15T05:19:20.745+00:00 2026-10-15T05:19:20.746976080Z INFO: 10.32.3.1:37564 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:19:27.698+00:00 2026-10-15T05:19:27.698776198Z INFO: 127.0.0.1:52116 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:19:30.745+00:00 2026-10-15T05:19:30.747139576Z INFO: 10.32.3.1:37230 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:19:40.747+00:00 2026-10-15T05:19:40.747233817Z INFO: 10.32.3.1:48412 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:19:50.747+00:00 2026-10-15T05:19:50.746985238Z INFO: 10.32.3.1:41220 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:20:00.746+00:00 2026-10-15T05:20:00.747586306Z INFO: 10.32.3.1:59588 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:20:07.948+00:00 2026-10-15T05:20:07.949379797Z INFO: 127.0.0.1:54766 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:20:10.744+00:00 2026-10-15T05:20:10.747103480Z INFO: 10.32.3.1:40408 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:20:20.745+00:00 2026-10-15T05:20:20.746536607Z INFO: 10.32.3.1:43004 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:20:30.745+00:00 2026-10-15T05:20:30.746837230Z INFO: 10.32.3.1:59800 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:20:32.447+00:00 2026-10-15T05:20:32.450365504Z [API_SERVER] No live session uses qwen-qwen3-8b; tearing its workers down +2026-10-15T05:20:40.746+00:00 2026-10-15T05:20:40.746710674Z INFO: 10.32.3.1:58932 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:20:50.745+00:00 2026-10-15T05:20:50.746442915Z INFO: 10.32.3.1:54538 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:20:58.177+00:00 2026-10-15T05:20:58.178160356Z INFO: 127.0.0.1:50924 - "POST /api/v1/telemetry HTTP/1.1" 200 OK +2026-10-15T05:21:00.745+00:00 2026-10-15T05:21:00.746518377Z INFO: 10.32.3.1:48928 - "GET /api/v1/healthz HTTP/1.1" 200 OK +2026-10-15T05:21:00.838+00:00 2026-10-15T05:21:00.842106688Z INFO: 127.0.0.1:50936 - "POST /api/v1/get_info HTTP/1.1" 200 OK +2026-10-15T05:21:04.579+00:00 2026-10-15T05:21:04.560065189Z [API_SERVER] enqueued op=forward_backward request_id=78e99e90-36d1-4203-863a-1cc73a5c1ee1 model_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 active_set=google/gemma-4-e2b-1 +2026-10-15T05:21:04.580+00:00 2026-10-15T05:21:04.560605605Z INFO: 127.0.0.1:50936 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK +2026-10-15T05:21:08.105+00:00 2026-10-15T05:21:08.106319850Z [API_SERVER] enqueued op=optim_step request_id=de504127-e705-4531-a2cc-f37ca7755809 model_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 active_set=google/gemma-4-e2b-1 +2026-10-15T05:21:08.105+00:00 2026-10-15T05:21:08.107759850Z INFO: 127.0.0.1:50936 - "POST /api/v1/optim_step HTTP/1.1" 200 OK +2026-10-15T05:21:10.744+00:00 2026-10-15T05:21:10.747068885Z INFO: 10.32.3.1:48390 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl new file mode 100644 index 00000000..9a091937 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl @@ -0,0 +1,7 @@ +{"ts": "2026-10-15T05:19:16.955+00:00", "t": 0.0, "phase": "start", "base_model": "google/gemma-4-e2b", "base_url": "http://127.0.0.1:9003"} +{"ts": "2026-10-15T05:19:16.956+00:00", "t": 0.001, "phase": "create_model.start"} +{"ts": "2026-10-15T05:21:00.658+00:00", "t": 103.703, "phase": "create_model.done", "model_id": "c0c2205d-aa11-49f3-9bfe-4dafd98e0314"} +{"ts": "2026-10-15T05:21:04.495+00:00", "t": 107.54, "phase": "forward_backward.start"} +{"ts": "2026-10-15T05:21:08.031+00:00", "t": 111.076, "phase": "forward_backward.done"} +{"ts": "2026-10-15T05:21:09.190+00:00", "t": 112.235, "phase": "optim_step.done"} +{"ts": "2026-10-15T05:21:09.196+00:00", "t": 112.241, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log new file mode 100644 index 00000000..8fae29fe --- /dev/null +++ b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log @@ -0,0 +1,8 @@ +[cold-start] 0.000s start {'base_model': 'google/gemma-4-e2b', 'base_url': 'http://127.0.0.1:9003'} +[cold-start] 0.001s create_model.start +[cold-start] 103.703s create_model.done {'model_id': 'c0c2205d-aa11-49f3-9bfe-4dafd98e0314'} +[cold-start] 107.540s forward_backward.start +[cold-start] 111.076s forward_backward.done +[cold-start] 112.235s optim_step.done +[cold-start] 112.241s end +EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt new file mode 100644 index 00000000..a7b79641 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt @@ -0,0 +1 @@ +2026-10-15T05:21:09.752+00:00 diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json new file mode 100644 index 00000000..89e9f845 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json @@ -0,0 +1,3265 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:10Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:10Z", + "name": "cold-start-fio.18de9a0ef295773d", + "namespace": "openrl-system", + "resourceVersion": "1792039090760383011", + "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5be63862", + "namespace": "openrl-system", + "resourceVersion": "1792039092515007000", + "uid": "355b59b9-3661-499d-8640-3f40c3c89375" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f5ecfa51a", + "namespace": "openrl-system", + "resourceVersion": "1792039092561663021", + "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:38:12Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:38:12Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:38:12Z", + "name": "cold-start-fio.18de9a0f61ddd1e4", + "namespace": "openrl-system", + "resourceVersion": "1792039092605935011", + "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:40:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039086935807002", + "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:40:41Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:40:41Z", + "name": "cold-start-fio.18de9a32081cf908", + "namespace": "openrl-system", + "resourceVersion": "1792039241446287009", + "uid": "14a62e42-13ba-4526-a04d-a934702e1163" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:18Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:18Z", + "message": "Pulling image \"ubuntu:24.04\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:18Z", + "name": "cold-start-fio.18de9a3a9867ce9f", + "namespace": "openrl-system", + "resourceVersion": "1792039278208207001", + "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad2bcac91", + "namespace": "openrl-system", + "resourceVersion": "1792039279188751014", + "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad378cd74", + "namespace": "openrl-system", + "resourceVersion": "1792039279219679006", + "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:41:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:41:19Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:41:19Z", + "name": "cold-start-fio.18de9a3ad5aed221", + "namespace": "openrl-system", + "resourceVersion": "1792039279235135013", + "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:11Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{fio}", + "kind": "Pod", + "name": "cold-start-fio", + "namespace": "openrl-system", + "resourceVersion": "1792039274774991002", + "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:11Z", + "message": "Stopping container fio", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:11Z", + "name": "cold-start-fio.18de9a70cd2c6a68", + "namespace": "openrl-system", + "resourceVersion": "1792039511048479001", + "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-645ff9777d-zg9cl", + "namespace": "openrl-system", + "resourceVersion": "1791941114537823021", + "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Stopping container api-server", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", + "namespace": "openrl-system", + "resourceVersion": "1792039545418735013", + "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-645ff9777d", + "namespace": "openrl-system", + "resourceVersion": "1792039545300351007", + "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", + "namespace": "openrl-system", + "resourceVersion": "1792039545387567001", + "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" + }, + "reason": "SuccessfulDelete", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546048591003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", + "namespace": "openrl-system", + "resourceVersion": "1792039546102351012", + "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:49Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:49Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", + "namespace": "openrl-system", + "resourceVersion": "1792039549712335019", + "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", + "namespace": "openrl-system", + "resourceVersion": "1792039558721215017", + "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", + "namespace": "openrl-system", + "resourceVersion": "1792039558783199013", + "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:58Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{api-server}", + "kind": "Pod", + "name": "open-rl-api-server-6d86b498f5-65swc", + "namespace": "openrl-system", + "resourceVersion": "1792039546075535003", + "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:58Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:58Z", + "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", + "namespace": "openrl-system", + "resourceVersion": "1792039558862303009", + "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:46Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "open-rl-api-server-6d86b498f5", + "namespace": "openrl-system", + "resourceVersion": "1792039545940671018", + "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:46Z", + "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:46Z", + "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", + "namespace": "openrl-system", + "resourceVersion": "1792039546068975009", + "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" + }, + "reason": "SuccessfulCreate", + "reportingComponent": "replicaset-controller", + "reportingInstance": "", + "source": { + "component": "replicaset-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545232655018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78c9668cf7", + "namespace": "openrl-system", + "resourceVersion": "1792039545327119005", + "uid": "7b1433df-3524-478b-aba2-8979950e245e" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:45:45Z", + "involvedObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "name": "open-rl-api-server", + "namespace": "openrl-system", + "resourceVersion": "1792039545464319018", + "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:45:45Z", + "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", + "metadata": { + "creationTimestamp": "2026-10-15T04:45:45Z", + "name": "open-rl-api-server.18de9a78ef8b9e1b", + "namespace": "openrl-system", + "resourceVersion": "1792039545956079020", + "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" + }, + "reason": "ScalingReplicaSet", + "reportingComponent": "deployment-controller", + "reportingInstance": "", + "source": { + "component": "deployment-controller" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:19:18Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-google-gemma-4-e2b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041557994719022", + "uid": "36500320-9722-4016-9c5f-922d8e1491bb" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:19:18Z", + "message": "Successfully assigned openrl-system/orw-lora-google-gemma-4-e2b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:19:18Z", + "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4d6d1f6dbc", + "namespace": "openrl-system", + "resourceVersion": "1792041558108079006", + "uid": "553af22f-384f-4e70-98ea-6ea30397ee54" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:19:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-google-gemma-4-e2b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041558091711022", + "uid": "36500320-9722-4016-9c5f-922d8e1491bb" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:19:21Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:19:21Z", + "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4e5228875d", + "namespace": "openrl-system", + "resourceVersion": "1792041561952783019", + "uid": "9305ffbb-2e08-43ce-95fe-8365fe6ffc17" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:19:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-google-gemma-4-e2b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041558091711022", + "uid": "36500320-9722-4016-9c5f-922d8e1491bb" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:19:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:19:21Z", + "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4e52be3a84", + "namespace": "openrl-system", + "resourceVersion": "1792041561974879003", + "uid": "9ff5c7e2-efe0-49ab-8b33-ac55b4ed4f9b" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:19:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-google-gemma-4-e2b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041558091711022", + "uid": "36500320-9722-4016-9c5f-922d8e1491bb" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:19:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:19:22Z", + "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4e576b76a1", + "namespace": "openrl-system", + "resourceVersion": "1792041562044143010", + "uid": "1a78c2b9-2440-496b-b893-a5460a0b0cf9" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:15Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875490623001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:15Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:15Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", + "namespace": "openrl-system", + "resourceVersion": "1792039875609279013", + "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:51:19Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:51:19Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:51:19Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", + "namespace": "openrl-system", + "resourceVersion": "1792039879126319008", + "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", + "namespace": "openrl-system", + "resourceVersion": "1792040061645247020", + "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:21Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:21Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:21Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", + "namespace": "openrl-system", + "resourceVersion": "1792040061849391002", + "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:54:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:54:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:54:22Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", + "namespace": "openrl-system", + "resourceVersion": "1792040062252079023", + "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792039875591167001", + "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", + "namespace": "openrl-system", + "resourceVersion": "1792040192780783021", + "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264403887001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", + "namespace": "openrl-system", + "resourceVersion": "1792040264444463021", + "uid": "a85006a2-68a6-4919-b148-08d58b546d07" + }, + "reason": "FailedScheduling", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Warning" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:44Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264434463001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:44Z", + "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", + "namespace": "openrl-system", + "resourceVersion": "1792040264884223002", + "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" + }, + "reason": "NotTriggerScaleUp", + "reportingComponent": "cluster-autoscaler", + "reportingInstance": "", + "source": { + "component": "cluster-autoscaler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:13Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040264893039001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:13Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:13Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", + "namespace": "openrl-system", + "resourceVersion": "1792040293171567001", + "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", + "namespace": "openrl-system", + "resourceVersion": "1792040296672127014", + "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", + "namespace": "openrl-system", + "resourceVersion": "1792040296703007017", + "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:58:16Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:58:16Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:58:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", + "namespace": "openrl-system", + "resourceVersion": "1792040296889487014", + "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040293156015001", + "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", + "namespace": "openrl-system", + "resourceVersion": "1792040427147711019", + "uid": "cc4da270-def2-456f-9374-372be8c717e4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:38Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498635999001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:38Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:38Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", + "namespace": "openrl-system", + "resourceVersion": "1792040498767471000", + "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", + "namespace": "openrl-system", + "resourceVersion": "1792040502270255017", + "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", + "namespace": "openrl-system", + "resourceVersion": "1792040502296943006", + "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:42Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:42Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:42Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", + "namespace": "openrl-system", + "resourceVersion": "1792040502475407023", + "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:09Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040498732799001", + "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:09Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:09Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", + "namespace": "openrl-system", + "resourceVersion": "1792040649924575012", + "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:52Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672220015012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:52Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:52Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", + "namespace": "openrl-system", + "resourceVersion": "1792039672350591019", + "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:47:56Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:47:56Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T04:47:56Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", + "namespace": "openrl-system", + "resourceVersion": "1792039676166239015", + "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", + "namespace": "openrl-system", + "resourceVersion": "1792039844457167000", + "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:44Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:44Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:44Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", + "namespace": "openrl-system", + "resourceVersion": "1792039844632591021", + "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:50:45Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:50:45Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:50:45Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", + "namespace": "openrl-system", + "resourceVersion": "1792039845170959016", + "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:56:32Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792039672311359012", + "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:56:32Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:56:32Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", + "namespace": "openrl-system", + "resourceVersion": "1792040192981183013", + "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:23Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040242926223012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:23Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:23Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", + "namespace": "openrl-system", + "resourceVersion": "1792040243043743024", + "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", + "namespace": "openrl-system", + "resourceVersion": "1792040246545487023", + "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", + "namespace": "openrl-system", + "resourceVersion": "1792040246576895006", + "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:57:26Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:57:26Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:57:26Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", + "namespace": "openrl-system", + "resourceVersion": "1792040246742815002", + "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:00:27Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040243022511012", + "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:00:27Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:00:27Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", + "namespace": "openrl-system", + "resourceVersion": "1792040427285631023", + "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:16Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476805407012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:16Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:16Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", + "namespace": "openrl-system", + "resourceVersion": "1792040476942623014", + "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", + "namespace": "openrl-system", + "resourceVersion": "1792040480747055005", + "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", + "namespace": "openrl-system", + "resourceVersion": "1792040480787967019", + "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:01:20Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:01:20Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:01:20Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", + "namespace": "openrl-system", + "resourceVersion": "1792040480963103007", + "uid": "d3158913-c79e-43d4-b468-d748641d49fe" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:10Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040476916127012", + "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:10Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:10Z", + "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", + "namespace": "openrl-system", + "resourceVersion": "1792040650191935010", + "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588098239023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", + "namespace": "openrl-system", + "resourceVersion": "1792038588224095005", + "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", + "namespace": "openrl-system", + "resourceVersion": "1792038591977631020", + "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:51Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:51Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:51Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", + "namespace": "openrl-system", + "resourceVersion": "1792038592011567002", + "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:29:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:29:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:29:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", + "namespace": "openrl-system", + "resourceVersion": "1792038592051215017", + "uid": "d431fad3-bafe-4002-8042-480b29aa499b" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792038588203679023", + "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", + "namespace": "openrl-system", + "resourceVersion": "1792038835732623016", + "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:58Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818213471023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:58Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:58Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", + "namespace": "openrl-system", + "resourceVersion": "1792040818411407005", + "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:07:01Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:07:01Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:07:01Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", + "namespace": "openrl-system", + "resourceVersion": "1792040821861695012", + "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", + "namespace": "openrl-system", + "resourceVersion": "1792040908347887024", + "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", + "namespace": "openrl-system", + "resourceVersion": "1792040908367439012", + "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:08:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:08:28Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:08:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", + "namespace": "openrl-system", + "resourceVersion": "1792040908418671004", + "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:10:57Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792040818390799023", + "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:10:57Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:10:57Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bd8cbffc810", + "namespace": "openrl-system", + "resourceVersion": "1792041057212047016", + "uid": "4a67fe5e-01b5-4b84-9453-b20887271ae4" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:18Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138729647023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:18Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:18Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bebcffc9f4c", + "namespace": "openrl-system", + "resourceVersion": "1792041138871999014", + "uid": "3e19d3e2-4122-4f9d-8324-8f74961c36bc" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9c8a8459", + "namespace": "openrl-system", + "resourceVersion": "1792041142294159019", + "uid": "aaaf55ef-ec50-4fa5-b6a7-28a27d836f8e" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9d08793e", + "namespace": "openrl-system", + "resourceVersion": "1792041142311631019", + "uid": "5ab187bd-863a-4013-9e2d-185b71c68ead" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:12:22Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:12:22Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:12:22Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9beca1f20c38", + "namespace": "openrl-system", + "resourceVersion": "1792041142412303013", + "uid": "193a6a12-3e5a-44e3-8d69-ea9d46ee4707" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:14:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041138843567023", + "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:14:49Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:14:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c0ec69a2183", + "namespace": "openrl-system", + "resourceVersion": "1792041289046591010", + "uid": "7161d68e-6e82-48a1-894f-7f149cadb6db" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:09Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369352463023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:09Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:09Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2181d762b3", + "namespace": "openrl-system", + "resourceVersion": "1792041369496943024", + "uid": "6a40425d-424d-438f-b398-45410b5817eb" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:13Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369459823023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:13Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:13Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c22606a262a", + "namespace": "openrl-system", + "resourceVersion": "1792041373214223010", + "uid": "c67940e2-268e-4a9a-94d4-05be2058728b" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:13Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369459823023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:13Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:13Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2260e74a63", + "namespace": "openrl-system", + "resourceVersion": "1792041373233151005", + "uid": "ecec41f0-1307-4892-ad43-57226155574a" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:16:13Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369459823023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:16:13Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:16:13Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2266080366", + "namespace": "openrl-system", + "resourceVersion": "1792041373308543020", + "uid": "6dc932f6-ae1f-48f6-b85d-e087521c8e20" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:18:28Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-sampler", + "namespace": "openrl-system", + "resourceVersion": "1792041369459823023", + "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:18:28Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:18:28Z", + "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c41f90718e5", + "namespace": "openrl-system", + "resourceVersion": "1792041508933663010", + "uid": "5c52845b-38a0-49b7-a3f5-f00f82aa7589" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:48Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528865007006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:48Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:48Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", + "namespace": "openrl-system", + "resourceVersion": "1792038529003663020", + "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", + "namespace": "openrl-system", + "resourceVersion": "1792038532754447004", + "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", + "namespace": "openrl-system", + "resourceVersion": "1792038532776847006", + "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:28:52Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:28:52Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T04:28:52Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", + "namespace": "openrl-system", + "resourceVersion": "1792038532827151022", + "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T04:33:55Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792038528965487006", + "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T04:33:55Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T04:33:55Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", + "namespace": "openrl-system", + "resourceVersion": "1792038835415695012", + "uid": "918abc9a-1f24-4767-8928-b202482f11ed" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:04:59Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699253647006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:04:59Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:04:59Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", + "namespace": "openrl-system", + "resourceVersion": "1792040699411615020", + "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:05:02Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:05:02Z", + "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", + "metadata": { + "creationTimestamp": "2026-10-15T05:05:02Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", + "namespace": "openrl-system", + "resourceVersion": "1792040702849391020", + "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" + }, + "reason": "Pulling", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", + "namespace": "openrl-system", + "resourceVersion": "1792040789347727015", + "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", + "namespace": "openrl-system", + "resourceVersion": "1792040789380863020", + "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:06:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:06:29Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:06:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", + "namespace": "openrl-system", + "resourceVersion": "1792040789417615006", + "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:10:57Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792040699366367006", + "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:10:57Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:10:57Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9bd8d38751ab", + "namespace": "openrl-system", + "resourceVersion": "1792041057319999002", + "uid": "0af06e74-e0c1-44de-a612-6fba47fb2b5c" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:46Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106258495006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:46Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:46Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be43ff3ded1", + "namespace": "openrl-system", + "resourceVersion": "1792041106381007003", + "uid": "4842f6c0-d606-444b-a970-6e9484189707" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50d27add6", + "namespace": "openrl-system", + "resourceVersion": "1792041109821407010", + "uid": "4e9749a9-e70a-453f-b51f-4d5ea7a60e86" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50daf2189", + "namespace": "openrl-system", + "resourceVersion": "1792041109840015005", + "uid": "2950a976-a558-471f-94fb-1ac680004e4c" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:11:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:11:49Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:11:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be5134afada", + "namespace": "openrl-system", + "resourceVersion": "1792041109931471020", + "uid": "d1c0b2ae-50a0-41e8-8333-441dd2250b23" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:14:49Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041106361663006", + "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:14:49Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:14:49Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c0ece29f9b7", + "namespace": "openrl-system", + "resourceVersion": "1792041289153695008", + "uid": "9db24059-5412-4998-8f1e-ff4723a02422" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:37Z", + "involvedObject": { + "apiVersion": "v1", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337881599006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:37Z", + "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:37Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1a2ddca97f", + "namespace": "openrl-system", + "resourceVersion": "1792041338007727010", + "uid": "622520b7-a0c6-4899-9d8e-8b7e289a5c19" + }, + "reason": "Scheduled", + "reportingComponent": "default-scheduler", + "reportingInstance": "", + "source": { + "component": "default-scheduler" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337986559006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:41Z", + "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:41Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afae88647", + "namespace": "openrl-system", + "resourceVersion": "1792041341445023014", + "uid": "174539ca-68ae-411a-9602-b94910a4f8a1" + }, + "reason": "Pulled", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337986559006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:41Z", + "message": "Container created", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:41Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afb72b4d7", + "namespace": "openrl-system", + "resourceVersion": "1792041341467327017", + "uid": "abc887ed-fbc2-4086-9892-21b9dbd9ef25" + }, + "reason": "Created", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:15:41Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337986559006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:15:41Z", + "message": "Container started", + "metadata": { + "creationTimestamp": "2026-10-15T05:15:41Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1b0091bf79", + "namespace": "openrl-system", + "resourceVersion": "1792041341538399007", + "uid": "ce445484-5b88-4ca9-a93d-806ea29b103f" + }, + "reason": "Started", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + }, + { + "apiVersion": "v1", + "count": 1, + "eventTime": null, + "firstTimestamp": "2026-10-15T05:18:29Z", + "involvedObject": { + "apiVersion": "v1", + "fieldPath": "spec.containers{worker}", + "kind": "Pod", + "name": "orw-lora-qwen-qwen3-8b-0-trainer", + "namespace": "openrl-system", + "resourceVersion": "1792041337986559006", + "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" + }, + "kind": "Event", + "lastTimestamp": "2026-10-15T05:18:29Z", + "message": "Stopping container worker", + "metadata": { + "creationTimestamp": "2026-10-15T05:18:29Z", + "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c41ffc5e74a", + "namespace": "openrl-system", + "resourceVersion": "1792041509032175007", + "uid": "aeee14b0-c22c-4f6d-b6c9-5f5dffbcc725" + }, + "reason": "Killing", + "reportingComponent": "kubelet", + "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", + "source": { + "component": "kubelet", + "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" + }, + "type": "Normal" + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt new file mode 100644 index 00000000..167c1377 --- /dev/null +++ b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt @@ -0,0 +1,32 @@ +2026-10-15T05:19:26.404504520Z +2026-10-15T05:19:26.404565293Z ================================================== +2026-10-15T05:19:26.404587125Z Open-RL PyTorch Training Worker +2026-10-15T05:19:26.404588917Z ================================================== +2026-10-15T05:19:26.404590495Z -> Hardware : CUDA_VISIBLE_DEVICES=ALL +2026-10-15T05:19:26.421803817Z -> Fine-Tuning Type: lora (Is LoRA: True) +2026-10-15T05:19:26.421826430Z +2026-10-15T05:19:26.629647334Z Loading base model google/gemma-4-e2b to cuda:0... +2026-10-15T05:19:27.261237071Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +2026-10-15T05:19:31.643943906Z [prefetch] reading 1 file(s), 9.54 GiB, from nfs into the page cache with 8 threads +2026-10-15T05:20:56.741663012Z [prefetch] done: 9.54 GiB in 85.1 s (115 MiB/s) +2026-10-15T05:20:58.914432483Z +Loading weights: 0%| | 0/1951 [00:00 "$R/$1.run.log" 2>&1; echo "$1 DONE=$? $(date -u +%T)" >> "$R/after-progress.txt"; } +run after-A1-0.5b-l4-cold Qwen/Qwen2.5-0.5B lvw8 2208 +run after-B1-0.5b-l4-cached Qwen/Qwen2.5-0.5B lvw8 2208 +run after-B2-0.5b-l4-cached Qwen/Qwen2.5-0.5B lvw8 2208 +run after-C1-8b-h100-cold Qwen/Qwen3-8B 1g3a 0853 +run after-D1-8b-h100-cached Qwen/Qwen3-8B 1g3a 0853 +run after-D2-8b-h100-cached Qwen/Qwen3-8B 1g3a 0853 +echo "ALL DONE $(date -u +%T)" >> "$R/after-progress.txt" diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt new file mode 100644 index 00000000..4c02098a --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt @@ -0,0 +1 @@ +file: /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json new file mode 100644 index 00000000..10888371 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039384, + "timestamp_ms" : 1792039384251, + "time" : "Thu Oct 15 04:43:04 2026", + "jobs" : [ + { + "jobname" : "randread", + "groupid" : 0, + "job_start" : 1792039354208, + "error" : 0, + "eta" : 0, + "elapsed" : 31, + "job options" : { + "name" : "randread", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", + "rw" : "randread", + "bs" : "4k", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "30", + "time_based" : "1" + }, + "read" : { + "io_bytes" : 4075520, + "io_kbytes" : 3980, + "bw_bytes" : 135714, + "bw" : 132, + "iops" : 33.133533, + "runtime" : 30030, + "total_ios" : 995, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 894793, + "max" : 576275926, + "mean" : 30175894.901508, + "stddev" : 43069178.925758, + "N" : 995, + "percentile" : { + "1.000000" : 913408, + "5.000000" : 962560, + "10.000000" : 995328, + "20.000000" : 1036288, + "30.000000" : 9764864, + "40.000000" : 20578304, + "50.000000" : 24772608, + "60.000000" : 29491200, + "70.000000" : 35389440, + "80.000000" : 43253760, + "90.000000" : 61603840, + "95.000000" : 77070336, + "99.000000" : 152043520, + "99.500000" : 240123904, + "99.900000" : 574619648, + "99.950000" : 574619648, + "99.990000" : 574619648 + } + }, + "lat_ns" : { + "min" : 895008, + "max" : 576276158, + "mean" : 30176112.192965, + "stddev" : 43069176.870365, + "N" : 995 + }, + "bw_min" : 16, + "bw_max" : 232, + "bw_agg" : 99.597683, + "bw_mean" : 132.533333, + "bw_dev" : 46.352908, + "bw_samples" : 60, + "iops_min" : 4, + "iops_max" : 58, + "iops_mean" : 33.133333, + "iops_stddev" : 11.588227, + "iops_samples" : 60 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 30029, + "usr_cpu" : 0.013320, + "sys_cpu" : 0.073263, + "ctx" : 996, + "majf" : 0, + "minf" : 9, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 11.356784 + }, + "latency_ms" : { + "2" : 17.185930, + "4" : 1.206030, + "10" : 0.301508, + "20" : 8.542714, + "50" : 45.427136, + "100" : 13.065327, + "250" : 2.512563, + "500" : 0.000000, + "750" : 0.402010, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json new file mode 100644 index 00000000..a7ac7b43 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json @@ -0,0 +1,261 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039447, + "timestamp_ms" : 1792039447569, + "time" : "Thu Oct 15 04:44:07 2026", + "jobs" : [ + { + "jobname" : "seqread-1M-4jobs", + "groupid" : 0, + "job_start" : 1792039417469, + "error" : 0, + "eta" : 0, + "elapsed" : 31, + "job options" : { + "name" : "seqread-1M-4jobs", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", + "rw" : "read", + "bs" : "1M", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "4", + "offset_increment" : "20%", + "runtime" : "30", + "time_based" : "1" + }, + "read" : { + "io_bytes" : 4250927104, + "io_kbytes" : 4151296, + "bw_bytes" : 141292531, + "bw" : 137980, + "iops" : 134.747058, + "runtime" : 30086, + "total_ios" : 4054, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 1172703, + "max" : 943811245, + "mean" : 29659560.486186, + "stddev" : 53937635.074449, + "N" : 4054, + "percentile" : { + "1.000000" : 1220608, + "5.000000" : 1269760, + "10.000000" : 1318912, + "20.000000" : 1400832, + "30.000000" : 1548288, + "40.000000" : 1957888, + "50.000000" : 2637824, + "60.000000" : 7438336, + "70.000000" : 27656192, + "80.000000" : 56360960, + "90.000000" : 96993280, + "95.000000" : 135266304, + "99.000000" : 217055232, + "99.500000" : 263192576, + "99.900000" : 541065216, + "99.950000" : 574619648, + "99.990000" : 943718400 + } + }, + "lat_ns" : { + "min" : 1172739, + "max" : 943811484, + "mean" : 29659692.227430, + "stddev" : 53937691.104136, + "N" : 4054 + }, + "bw_min" : 10240, + "bw_max" : 1286144, + "bw_agg" : 100.000000, + "bw_mean" : 138337.771751, + "bw_dev" : 90992.521392, + "bw_samples" : 238, + "iops_min" : 10, + "iops_max" : 1256, + "iops_mean" : 135.095480, + "iops_stddev" : 88.859884, + "iops_samples" : 238 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 120255, + "usr_cpu" : 0.007484, + "sys_cpu" : 0.212049, + "ctx" : 4064, + "majf" : 0, + "minf" : 1061, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_ms" : { + "2" : 40.083868, + "4" : 13.394179, + "10" : 11.198816, + "20" : 3.305377, + "50" : 10.039467, + "100" : 12.629502, + "250" : 8.658115, + "500" : 0.518007, + "750" : 0.148002, + "1000" : 0.024667, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json new file mode 100644 index 00000000..5f1e6a58 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039509, + "timestamp_ms" : 1792039509047, + "time" : "Thu Oct 15 04:45:09 2026", + "jobs" : [ + { + "jobname" : "seqread-buffered", + "groupid" : 0, + "job_start" : 1792039448961, + "error" : 0, + "eta" : 0, + "elapsed" : 61, + "job options" : { + "name" : "seqread-buffered", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", + "rw" : "read", + "bs" : "1M", + "direct" : "0", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "60", + "time_based" : "0" + }, + "read" : { + "io_bytes" : 3805282304, + "io_kbytes" : 3716096, + "bw_bytes" : 63342193, + "bw" : 61857, + "iops" : 60.407824, + "runtime" : 60075, + "total_ios" : 3629, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 100093, + "max" : 1184710687, + "mean" : 16551942.435382, + "stddev" : 46828063.988788, + "N" : 3629, + "percentile" : { + "1.000000" : 119296, + "5.000000" : 152576, + "10.000000" : 276480, + "20.000000" : 460800, + "30.000000" : 692224, + "40.000000" : 798720, + "50.000000" : 954368, + "60.000000" : 1089536, + "70.000000" : 1318912, + "80.000000" : 11337728, + "90.000000" : 58458112, + "95.000000" : 99090432, + "99.000000" : 179306496, + "99.500000" : 229638144, + "99.900000" : 591396864, + "99.950000" : 633339904, + "99.990000" : 1182793728 + } + }, + "lat_ns" : { + "min" : 100311, + "max" : 1184711128, + "mean" : 16552079.251309, + "stddev" : 46828089.869993, + "N" : 3629 + }, + "bw_min" : 2048, + "bw_max" : 1347584, + "bw_agg" : 100.000000, + "bw_mean" : 62438.184874, + "bw_dev" : 202926.862845, + "bw_samples" : 119, + "iops_min" : 2, + "iops_max" : 1316, + "iops_mean" : 60.974790, + "iops_stddev" : 198.170764, + "iops_samples" : 119 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 60074, + "usr_cpu" : 0.019975, + "sys_cpu" : 0.759064, + "ctx" : 3357, + "majf" : 0, + "minf" : 268, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 8.845412, + "500" : 13.585010, + "750" : 11.380546, + "1000" : 20.088179 + }, + "latency_ms" : { + "2" : 19.977955, + "4" : 1.598236, + "10" : 4.298705, + "20" : 2.011573, + "50" : 6.200055, + "100" : 7.109397, + "250" : 4.519151, + "500" : 0.275558, + "750" : 0.082667, + "1000" : 0.000000, + "2000" : 0.027556, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json new file mode 100644 index 00000000..943ffb14 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039352, + "timestamp_ms" : 1792039352765, + "time" : "Thu Oct 15 04:42:32 2026", + "jobs" : [ + { + "jobname" : "seqread", + "groupid" : 0, + "job_start" : 1792039292506, + "error" : 0, + "eta" : 0, + "elapsed" : 61, + "job options" : { + "name" : "seqread", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", + "rw" : "read", + "bs" : "1M", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "60", + "time_based" : "0" + }, + "read" : { + "io_bytes" : 1023410176, + "io_kbytes" : 999424, + "bw_bytes" : 16986342, + "bw" : 16588, + "iops" : 16.199439, + "runtime" : 60249, + "total_ios" : 976, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 5615855, + "max" : 738292284, + "mean" : 61723383.765369, + "stddev" : 60353873.274478, + "N" : 976, + "percentile" : { + "1.000000" : 6324224, + "5.000000" : 6914048, + "10.000000" : 7569408, + "20.000000" : 10027008, + "30.000000" : 26345472, + "40.000000" : 35913728, + "50.000000" : 47972352, + "60.000000" : 61603840, + "70.000000" : 76021760, + "80.000000" : 100139008, + "90.000000" : 128450560, + "95.000000" : 162529280, + "99.000000" : 238026752, + "99.500000" : 371195904, + "99.900000" : 742391808, + "99.950000" : 742391808, + "99.990000" : 742391808 + } + }, + "lat_ns" : { + "min" : 5616100, + "max" : 738292769, + "mean" : 61723666.015369, + "stddev" : 60353875.885652, + "N" : 976 + }, + "bw_min" : 4096, + "bw_max" : 36864, + "bw_agg" : 100.000000, + "bw_mean" : 16640.000000, + "bw_dev" : 6235.464846, + "bw_samples" : 120, + "iops_min" : 4, + "iops_max" : 36, + "iops_mean" : 16.250000, + "iops_stddev" : 6.089321, + "iops_samples" : 120 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 60248, + "usr_cpu" : 0.006639, + "sys_cpu" : 0.129465, + "ctx" : 978, + "majf" : 0, + "minf" : 269, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_ms" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 19.774590, + "20" : 6.762295, + "50" : 25.102459, + "100" : 28.073770, + "250" : 19.467213, + "500" : 0.614754, + "750" : 0.204918, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json new file mode 100644 index 00000000..430aa12a --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039416, + "timestamp_ms" : 1792039416020, + "time" : "Thu Oct 15 04:43:36 2026", + "jobs" : [ + { + "jobname" : "seqread-256k-qd1", + "groupid" : 0, + "job_start" : 1792039385969, + "error" : 0, + "eta" : 0, + "elapsed" : 31, + "job options" : { + "name" : "seqread-256k-qd1", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", + "rw" : "read", + "bs" : "256k", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "30", + "time_based" : "1" + }, + "read" : { + "io_bytes" : 1409810432, + "io_kbytes" : 1376768, + "bw_bytes" : 46945171, + "bw" : 45844, + "iops" : 179.081616, + "runtime" : 30031, + "total_ios" : 5378, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 985502, + "max" : 381577086, + "mean" : 5583040.162886, + "stddev" : 15453629.349114, + "N" : 5378, + "percentile" : { + "1.000000" : 1011712, + "5.000000" : 1044480, + "10.000000" : 1073152, + "20.000000" : 1105920, + "30.000000" : 1138688, + "40.000000" : 1171456, + "50.000000" : 1236992, + "60.000000" : 1335296, + "70.000000" : 1679360, + "80.000000" : 2473984, + "90.000000" : 6979584, + "95.000000" : 34340864, + "99.000000" : 76021760, + "99.500000" : 92798976, + "99.900000" : 147849216, + "99.950000" : 158334976, + "99.990000" : 379584512 + } + }, + "lat_ns" : { + "min" : 985546, + "max" : 381577157, + "mean" : 5583109.203979, + "stddev" : 15453645.045346, + "N" : 5378 + }, + "bw_min" : 5632, + "bw_max" : 229888, + "bw_agg" : 100.000000, + "bw_mean" : 45883.733333, + "bw_dev" : 70722.254239, + "bw_samples" : 60, + "iops_min" : 22, + "iops_max" : 898, + "iops_mean" : 179.233333, + "iops_stddev" : 276.258806, + "iops_samples" : 60 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 30030, + "usr_cpu" : 0.043290, + "sys_cpu" : 0.326340, + "ctx" : 5379, + "majf" : 0, + "minf" : 74, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.185943 + }, + "latency_ms" : { + "2" : 71.309037, + "4" : 18.185199, + "10" : 0.409074, + "20" : 1.283005, + "50" : 5.931573, + "100" : 2.361473, + "250" : 0.316103, + "500" : 0.018594, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/monitoring.md b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/monitoring.md new file mode 100644 index 00000000..45d2aa87 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/monitoring.md @@ -0,0 +1,11 @@ +| minute ending (UTC) | read MiB/s | read ops/s | write MiB/s | +|---|---|---|---| +| 2026-10-15T04:38:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:39:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:40:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:41:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:42:00Z | 0.0 | 5 | 0.0 | +| 2026-10-15T04:43:00Z | 6.8 | 100 | 0.0 | +| 2026-10-15T04:44:00Z | 9.5 | 63 | 0.0 | +| 2026-10-15T04:45:00Z | 19.1 | 210 | 0.0 | +| 2026-10-15T04:46:00Z | 22.0 | 0 | 0.0 | diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt new file mode 100644 index 00000000..191b93ff --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt @@ -0,0 +1,31 @@ +device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 + opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none + age: 232 + caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 + sec: flavor=1,pseudoflavor=1 + events: 31 451 0 0 13 9 518 0 0 3630 0 0 4 0 11 0 0 11 0 0 0 0 0 0 0 0 0 + bytes: 3805282304 0 6688223232 0 10494554112 0 929280 0 + RPC iostats version: 1.1 p/v: 100003/3 (nfs) + xprt: tcp 961 1 2 0 1 15104 15104 0 30403 0 4 13 15292 + per-op statistics + NULL: 1 1 0 44 24 3 1 4 0 + GETATTR: 33 33 0 4720 3696 0 34 34 0 + SETATTR: 0 0 0 0 0 0 0 0 0 + LOOKUP: 17 17 0 3100 4012 0 16 17 0 + ACCESS: 9 9 0 1324 1080 0 9 9 0 + READLINK: 3 3 0 432 588 0 38 38 0 + READ: 15033 15033 0 2345148 10496478336 206 346790 347214 0 + WRITE: 0 0 0 0 0 0 0 0 0 + CREATE: 0 0 0 0 0 0 0 0 0 + MKDIR: 0 0 0 0 0 0 0 0 0 + SYMLINK: 0 0 0 0 0 0 0 0 0 + MKNOD: 0 0 0 0 0 0 0 0 0 + REMOVE: 0 0 0 0 0 0 0 0 0 + RMDIR: 0 0 0 0 0 0 0 0 0 + RENAME: 0 0 0 0 0 0 0 0 0 + LINK: 0 0 0 0 0 0 0 0 0 + READDIR: 0 0 0 0 0 0 0 0 0 + READDIRPLUS: 2 2 0 336 2764 0 2 2 0 + FSSTAT: 2 2 0 272 168 0 1 1 0 + FSINFO: 2 2 0 272 160 0 2 2 0 + PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt new file mode 100644 index 00000000..499e4613 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt @@ -0,0 +1,32 @@ +lrwxrwxrwx 1 root root 76 Sep 17 20:59 /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors -> ../../blobs/75311d91bb08cf0b882913da464a1e722a31fb44db35208663487efb7a3d8ed6 +device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 + opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none + age: 13 + caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 + sec: flavor=1,pseudoflavor=1 + events: 1 72 0 0 2 9 89 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + bytes: 0 0 0 0 0 0 0 0 + RPC iostats version: 1.1 p/v: 100003/3 (nfs) + xprt: tcp 961 1 2 0 1 30 30 0 30 0 2 0 0 + per-op statistics + NULL: 1 1 0 44 24 3 1 4 0 + GETATTR: 3 3 0 416 336 0 3 3 0 + SETATTR: 0 0 0 0 0 0 0 0 0 + LOOKUP: 9 9 0 1596 2124 0 8 8 0 + ACCESS: 8 8 0 1176 960 0 8 9 0 + READLINK: 3 3 0 432 588 0 38 38 0 + READ: 0 0 0 0 0 0 0 0 0 + WRITE: 0 0 0 0 0 0 0 0 0 + CREATE: 0 0 0 0 0 0 0 0 0 + MKDIR: 0 0 0 0 0 0 0 0 0 + SYMLINK: 0 0 0 0 0 0 0 0 0 + MKNOD: 0 0 0 0 0 0 0 0 0 + REMOVE: 0 0 0 0 0 0 0 0 0 + RMDIR: 0 0 0 0 0 0 0 0 0 + RENAME: 0 0 0 0 0 0 0 0 0 + LINK: 0 0 0 0 0 0 0 0 0 + READDIR: 0 0 0 0 0 0 0 0 0 + READDIRPLUS: 2 2 0 336 2764 0 2 2 0 + FSSTAT: 0 0 0 0 0 0 0 0 0 + FSINFO: 2 2 0 272 160 0 2 2 0 + PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/summary.md b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/summary.md new file mode 100644 index 00000000..361873b4 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/summary.md @@ -0,0 +1,11 @@ +# fio from fio-h100-1g3a-coldfile + +file: file: /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors + +| job | MiB/s | IOPS | mean latency ms | read | seconds | +|---|---|---|---|---|---| +| fio-seqread-1M-direct | 16.2 | 16 | 61.72 | 0.95 GiB | 60.2 | +| fio-randread-4k-direct | 0.1 | 33 | 30.18 | 0.00 GiB | 30.0 | +| fio-seqread-256k-qd1-direct | 44.8 | 179 | 5.58 | 1.31 GiB | 30.0 | +| fio-seqread-1M-4jobs-direct | 134.7 | 135 | 29.66 | 3.96 GiB | 30.1 | +| fio-seqread-1M-buffered | 60.4 | 60 | 16.55 | 3.54 GiB | 60.1 | diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt b/runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt new file mode 100644 index 00000000..9687ad99 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt @@ -0,0 +1 @@ +file: /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json new file mode 100644 index 00000000..e80f7dbb --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039170, + "timestamp_ms" : 1792039170876, + "time" : "Thu Oct 15 04:39:30 2026", + "jobs" : [ + { + "jobname" : "randread", + "groupid" : 0, + "job_start" : 1792039140863, + "error" : 0, + "eta" : 0, + "elapsed" : 31, + "job options" : { + "name" : "randread", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", + "rw" : "randread", + "bs" : "4k", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "30", + "time_based" : "1" + }, + "read" : { + "io_bytes" : 131043328, + "io_kbytes" : 127972, + "bw_bytes" : 4367965, + "bw" : 4265, + "iops" : 1066.397787, + "runtime" : 30001, + "total_ios" : 31993, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 809541, + "max" : 4640366, + "mean" : 936167.207389, + "stddev" : 85513.990026, + "N" : 31993, + "percentile" : { + "1.000000" : 839680, + "5.000000" : 856064, + "10.000000" : 872448, + "20.000000" : 880640, + "30.000000" : 897024, + "40.000000" : 905216, + "50.000000" : 921600, + "60.000000" : 937984, + "70.000000" : 954368, + "80.000000" : 978944, + "90.000000" : 1011712, + "95.000000" : 1056768, + "99.000000" : 1155072, + "99.500000" : 1204224, + "99.900000" : 1646592, + "99.950000" : 2277376, + "99.990000" : 3719168 + } + }, + "lat_ns" : { + "min" : 809614, + "max" : 4640441, + "mean" : 936290.783265, + "stddev" : 85520.922977, + "N" : 31993 + }, + "bw_min" : 3896, + "bw_max" : 4408, + "bw_agg" : 99.986149, + "bw_mean" : 4265.898305, + "bw_dev" : 104.891026, + "bw_samples" : 59, + "iops_min" : 974, + "iops_max" : 1102, + "iops_mean" : 1066.474576, + "iops_stddev" : 26.222757, + "iops_samples" : 59 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 30000, + "usr_cpu" : 0.253333, + "sys_cpu" : 1.473333, + "ctx" : 31994, + "majf" : 0, + "minf" : 11, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 86.890882 + }, + "latency_ms" : { + "2" : 13.049730, + "4" : 0.050011, + "10" : 0.010000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json new file mode 100644 index 00000000..9dfff85b --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json @@ -0,0 +1,261 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039233, + "timestamp_ms" : 1792039233725, + "time" : "Thu Oct 15 04:40:33 2026", + "jobs" : [ + { + "jobname" : "seqread-1M-4jobs", + "groupid" : 0, + "job_start" : 1792039203706, + "error" : 0, + "eta" : 0, + "elapsed" : 31, + "job options" : { + "name" : "seqread-1M-4jobs", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", + "rw" : "read", + "bs" : "1M", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "4", + "offset_increment" : "20%", + "runtime" : "30", + "time_based" : "1" + }, + "read" : { + "io_bytes" : 55853449216, + "io_kbytes" : 54544384, + "bw_bytes" : 1861595480, + "bw" : 1817964, + "iops" : 1775.355798, + "runtime" : 30003, + "total_ios" : 53266, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 1526779, + "max" : 21354553, + "mean" : 2249205.334566, + "stddev" : 1263897.589092, + "N" : 53266, + "percentile" : { + "1.000000" : 1859584, + "5.000000" : 2007040, + "10.000000" : 2039808, + "20.000000" : 2072576, + "30.000000" : 2088960, + "40.000000" : 2113536, + "50.000000" : 2113536, + "60.000000" : 2146304, + "70.000000" : 2146304, + "80.000000" : 2179072, + "90.000000" : 2211840, + "95.000000" : 2244608, + "99.000000" : 5210112, + "99.500000" : 15138816, + "99.900000" : 19005440, + "99.950000" : 19791872, + "99.990000" : 20316160 + } + }, + "lat_ns" : { + "min" : 1526935, + "max" : 21354747, + "mean" : 2249392.502103, + "stddev" : 1263898.426861, + "N" : 53266 + }, + "bw_min" : 1267712, + "bw_max" : 1939456, + "bw_agg" : 100.000000, + "bw_mean" : 1819144.677966, + "bw_dev" : 38366.646123, + "bw_samples" : 236, + "iops_min" : 1238, + "iops_max" : 1894, + "iops_mean" : 1776.508475, + "iops_stddev" : 37.467428, + "iops_samples" : 236 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 120004, + "usr_cpu" : 0.233326, + "sys_cpu" : 3.800707, + "ctx" : 53319, + "majf" : 0, + "minf" : 1077, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_ms" : { + "2" : 4.126460, + "4" : 94.531221, + "10" : 0.563211, + "20" : 0.741561, + "50" : 0.037547, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json new file mode 100644 index 00000000..b701ea15 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039239, + "timestamp_ms" : 1792039239725, + "time" : "Thu Oct 15 04:40:39 2026", + "jobs" : [ + { + "jobname" : "seqread-buffered", + "groupid" : 0, + "job_start" : 1792039235446, + "error" : 0, + "eta" : 0, + "elapsed" : 5, + "job options" : { + "name" : "seqread-buffered", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", + "rw" : "read", + "bs" : "1M", + "direct" : "0", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "60", + "time_based" : "0" + }, + "read" : { + "io_bytes" : 3996123136, + "io_kbytes" : 3902464, + "bw_bytes" : 936737725, + "bw" : 914782, + "iops" : 893.342710, + "runtime" : 4266, + "total_ios" : 3811, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 139007, + "max" : 19689357, + "mean" : 1117032.838625, + "stddev" : 1439021.300843, + "N" : 3811, + "percentile" : { + "1.000000" : 276480, + "5.000000" : 313344, + "10.000000" : 346112, + "20.000000" : 432128, + "30.000000" : 774144, + "40.000000" : 864256, + "50.000000" : 978944, + "60.000000" : 1089536, + "70.000000" : 1220608, + "80.000000" : 1384448, + "90.000000" : 1531904, + "95.000000" : 1613824, + "99.000000" : 8847360, + "99.500000" : 13303808, + "99.900000" : 18743296, + "99.950000" : 19267584, + "99.990000" : 19791872 + } + }, + "lat_ns" : { + "min" : 139199, + "max" : 19689473, + "mean" : 1117200.393597, + "stddev" : 1439021.295617, + "N" : 3811 + }, + "bw_min" : 679936, + "bw_max" : 1136640, + "bw_agg" : 98.674337, + "bw_mean" : 902656.000000, + "bw_dev" : 179747.109310, + "bw_samples" : 8, + "iops_min" : 664, + "iops_max" : 1110, + "iops_mean" : 881.500000, + "iops_stddev" : 175.534286, + "iops_samples" : 8 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 4265, + "usr_cpu" : 0.257913, + "sys_cpu" : 12.942556, + "ctx" : 3811, + "majf" : 0, + "minf" : 265, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.341118, + "500" : 22.986093, + "750" : 5.484125, + "1000" : 22.644975 + }, + "latency_ms" : { + "2" : 45.945946, + "4" : 1.049593, + "10" : 0.682236, + "20" : 0.865914, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json new file mode 100644 index 00000000..33e8c57a --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039139, + "timestamp_ms" : 1792039139572, + "time" : "Thu Oct 15 04:38:59 2026", + "jobs" : [ + { + "jobname" : "seqread", + "groupid" : 0, + "job_start" : 1792039131591, + "error" : 0, + "eta" : 0, + "elapsed" : 9, + "job options" : { + "name" : "seqread", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", + "rw" : "read", + "bs" : "1M", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "60", + "time_based" : "0" + }, + "read" : { + "io_bytes" : 3996123136, + "io_kbytes" : 3902464, + "bw_bytes" : 501962458, + "bw" : 490197, + "iops" : 478.708705, + "runtime" : 7961, + "total_ios" : 3811, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 1666933, + "max" : 8936381, + "mean" : 2086845.958016, + "stddev" : 478355.966195, + "N" : 3811, + "percentile" : { + "1.000000" : 1728512, + "5.000000" : 1761280, + "10.000000" : 1777664, + "20.000000" : 1810432, + "30.000000" : 1859584, + "40.000000" : 1892352, + "50.000000" : 1941504, + "60.000000" : 1990656, + "70.000000" : 2072576, + "80.000000" : 2211840, + "90.000000" : 2506752, + "95.000000" : 2998272, + "99.000000" : 4177920, + "99.500000" : 4685824, + "99.900000" : 5603328, + "99.950000" : 7241728, + "99.990000" : 8978432 + } + }, + "lat_ns" : { + "min" : 1667036, + "max" : 8937220, + "mean" : 2086990.124902, + "stddev" : 478357.297592, + "N" : 3811 + }, + "bw_min" : 348160, + "bw_max" : 542720, + "bw_agg" : 100.000000, + "bw_mean" : 491383.466667, + "bw_dev" : 62462.641047, + "bw_samples" : 15, + "iops_min" : 340, + "iops_max" : 530, + "iops_mean" : 479.866667, + "iops_stddev" : 60.998673, + "iops_samples" : 15 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 7960, + "usr_cpu" : 0.113065, + "sys_cpu" : 3.994975, + "ctx" : 3813, + "majf" : 0, + "minf" : 268, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_ms" : { + "2" : 60.299134, + "4" : 38.362634, + "10" : 1.338231, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json new file mode 100644 index 00000000..9fdd891d --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json @@ -0,0 +1,260 @@ +{ + "fio version" : "fio-3.36", + "timestamp" : 1792039202, + "timestamp_ms" : 1792039202382, + "time" : "Thu Oct 15 04:40:02 2026", + "jobs" : [ + { + "jobname" : "seqread-256k-qd1", + "groupid" : 0, + "job_start" : 1792039172362, + "error" : 0, + "eta" : 0, + "elapsed" : 31, + "job options" : { + "name" : "seqread-256k-qd1", + "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", + "rw" : "read", + "bs" : "256k", + "direct" : "1", + "ioengine" : "psync", + "numjobs" : "1", + "runtime" : "30", + "time_based" : "1" + }, + "read" : { + "io_bytes" : 6612844544, + "io_kbytes" : 6457856, + "bw_bytes" : 220420804, + "bw" : 215254, + "iops" : 840.838639, + "runtime" : 30001, + "total_ios" : 25226, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 1035984, + "max" : 4418595, + "mean" : 1187880.284825, + "stddev" : 82024.012557, + "N" : 25226, + "percentile" : { + "1.000000" : 1073152, + "5.000000" : 1089536, + "10.000000" : 1105920, + "20.000000" : 1138688, + "30.000000" : 1155072, + "40.000000" : 1155072, + "50.000000" : 1171456, + "60.000000" : 1187840, + "70.000000" : 1204224, + "80.000000" : 1236992, + "90.000000" : 1269760, + "95.000000" : 1318912, + "99.000000" : 1417216, + "99.500000" : 1466368, + "99.900000" : 1662976, + "99.950000" : 2211840, + "99.990000" : 3424256 + } + }, + "lat_ns" : { + "min" : 1036155, + "max" : 4418682, + "mean" : 1188002.449060, + "stddev" : 82030.606240, + "N" : 25226 + }, + "bw_min" : 202240, + "bw_max" : 223232, + "bw_agg" : 100.000000, + "bw_mean" : 215430.508475, + "bw_dev" : 4740.794515, + "bw_samples" : 59, + "iops_min" : 790, + "iops_max" : 872, + "iops_mean" : 841.525424, + "iops_stddev" : 18.518729, + "iops_samples" : 59 + }, + "write" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "trim" : { + "io_bytes" : 0, + "io_kbytes" : 0, + "bw_bytes" : 0, + "bw" : 0, + "iops" : 0.000000, + "runtime" : 0, + "total_ios" : 0, + "short_ios" : 0, + "drop_ios" : 0, + "slat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "clat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + }, + "bw_min" : 0, + "bw_max" : 0, + "bw_agg" : 0.000000, + "bw_mean" : 0.000000, + "bw_dev" : 0.000000, + "bw_samples" : 0, + "iops_min" : 0, + "iops_max" : 0, + "iops_mean" : 0.000000, + "iops_stddev" : 0.000000, + "iops_samples" : 0 + }, + "sync" : { + "total_ios" : 0, + "lat_ns" : { + "min" : 0, + "max" : 0, + "mean" : 0.000000, + "stddev" : 0.000000, + "N" : 0 + } + }, + "job_runtime" : 30000, + "usr_cpu" : 0.253333, + "sys_cpu" : 2.500000, + "ctx" : 25230, + "majf" : 0, + "minf" : 75, + "iodepth_level" : { + "1" : 100.000000, + "2" : 0.000000, + "4" : 0.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_submit" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "iodepth_complete" : { + "0" : 0.000000, + "4" : 100.000000, + "8" : 0.000000, + "16" : 0.000000, + "32" : 0.000000, + "64" : 0.000000, + ">=64" : 0.000000 + }, + "latency_ns" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_us" : { + "2" : 0.000000, + "4" : 0.000000, + "10" : 0.000000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000 + }, + "latency_ms" : { + "2" : 99.944502, + "4" : 0.051534, + "10" : 0.010000, + "20" : 0.000000, + "50" : 0.000000, + "100" : 0.000000, + "250" : 0.000000, + "500" : 0.000000, + "750" : 0.000000, + "1000" : 0.000000, + "2000" : 0.000000, + ">=2000" : 0.000000 + }, + "latency_depth" : 1, + "latency_target" : 0, + "latency_percentile" : 100.000000, + "latency_window" : 0 + } + ] +} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json new file mode 100644 index 00000000..69d2783b --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json @@ -0,0 +1,302 @@ +{ + "read_bytes_count": { + "timeSeries": [ + { + "metric": { + "labels": { + "file_share": "vol1" + }, + "type": "file.googleapis.com/nfs/server/read_bytes_count" + }, + "resource": { + "type": "filestore_instance", + "labels": { + "location": "us-central1-b", + "project_id": "cdrollouts-sunilarora", + "instance_name": "pvc-b69fc9c0-172e-4423-a7bf-abc5d69a95c1" + } + }, + "metricKind": "GAUGE", + "valueType": "DOUBLE", + "points": [ + { + "interval": { + "startTime": "2026-10-15T04:46:00Z", + "endTime": "2026-10-15T04:46:00Z" + }, + "value": { + "doubleValue": 23058538.966666665 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:45:00Z", + "endTime": "2026-10-15T04:45:00Z" + }, + "value": { + "doubleValue": 19997416.133333333 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:44:00Z", + "endTime": "2026-10-15T04:44:00Z" + }, + "value": { + "doubleValue": 9963643.033333333 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:43:00Z", + "endTime": "2026-10-15T04:43:00Z" + }, + "value": { + "doubleValue": 7131094.833333333 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:42:00Z", + "endTime": "2026-10-15T04:42:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:41:00Z", + "endTime": "2026-10-15T04:41:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:40:00Z", + "endTime": "2026-10-15T04:40:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:39:00Z", + "endTime": "2026-10-15T04:39:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:38:00Z", + "endTime": "2026-10-15T04:38:00Z" + }, + "value": { + "doubleValue": 0 + } + } + ] + } + ], + "unit": "By/s" + }, + "read_ops_count": { + "timeSeries": [ + { + "metric": { + "labels": { + "file_share": "vol1" + }, + "type": "file.googleapis.com/nfs/server/read_ops_count" + }, + "resource": { + "type": "filestore_instance", + "labels": { + "project_id": "cdrollouts-sunilarora", + "location": "us-central1-b", + "instance_name": "pvc-b69fc9c0-172e-4423-a7bf-abc5d69a95c1" + } + }, + "metricKind": "GAUGE", + "valueType": "DOUBLE", + "points": [ + { + "interval": { + "startTime": "2026-10-15T04:45:00Z", + "endTime": "2026-10-15T04:45:00Z" + }, + "value": { + "doubleValue": 210.05 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:44:00Z", + "endTime": "2026-10-15T04:44:00Z" + }, + "value": { + "doubleValue": 63.083333333333336 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:43:00Z", + "endTime": "2026-10-15T04:43:00Z" + }, + "value": { + "doubleValue": 100.35 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:42:00Z", + "endTime": "2026-10-15T04:42:00Z" + }, + "value": { + "doubleValue": 4.55 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:41:00Z", + "endTime": "2026-10-15T04:41:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:40:00Z", + "endTime": "2026-10-15T04:40:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:39:00Z", + "endTime": "2026-10-15T04:39:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:38:00Z", + "endTime": "2026-10-15T04:38:00Z" + }, + "value": { + "doubleValue": 0 + } + } + ] + } + ], + "unit": "{operation}/s" + }, + "write_bytes_count": { + "timeSeries": [ + { + "metric": { + "labels": { + "file_share": "vol1" + }, + "type": "file.googleapis.com/nfs/server/write_bytes_count" + }, + "resource": { + "type": "filestore_instance", + "labels": { + "instance_name": "pvc-b69fc9c0-172e-4423-a7bf-abc5d69a95c1", + "project_id": "cdrollouts-sunilarora", + "location": "us-central1-b" + } + }, + "metricKind": "GAUGE", + "valueType": "DOUBLE", + "points": [ + { + "interval": { + "startTime": "2026-10-15T04:45:00Z", + "endTime": "2026-10-15T04:45:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:44:00Z", + "endTime": "2026-10-15T04:44:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:43:00Z", + "endTime": "2026-10-15T04:43:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:42:00Z", + "endTime": "2026-10-15T04:42:00Z" + }, + "value": { + "doubleValue": 273.06666666666666 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:41:00Z", + "endTime": "2026-10-15T04:41:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:40:00Z", + "endTime": "2026-10-15T04:40:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:39:00Z", + "endTime": "2026-10-15T04:39:00Z" + }, + "value": { + "doubleValue": 0 + } + }, + { + "interval": { + "startTime": "2026-10-15T04:38:00Z", + "endTime": "2026-10-15T04:38:00Z" + }, + "value": { + "doubleValue": 0 + } + } + ] + } + ], + "unit": "By/s" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.md b/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.md new file mode 100644 index 00000000..45d2aa87 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.md @@ -0,0 +1,11 @@ +| minute ending (UTC) | read MiB/s | read ops/s | write MiB/s | +|---|---|---|---| +| 2026-10-15T04:38:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:39:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:40:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:41:00Z | 0.0 | 0 | 0.0 | +| 2026-10-15T04:42:00Z | 0.0 | 5 | 0.0 | +| 2026-10-15T04:43:00Z | 6.8 | 100 | 0.0 | +| 2026-10-15T04:44:00Z | 9.5 | 63 | 0.0 | +| 2026-10-15T04:45:00Z | 19.1 | 210 | 0.0 | +| 2026-10-15T04:46:00Z | 22.0 | 0 | 0.0 | diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt b/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt new file mode 100644 index 00000000..d17e52aa --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt @@ -0,0 +1,31 @@ +device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 + opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none + age: 150 + caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 + sec: flavor=1,pseudoflavor=1 + events: 52 699 0 0 35 11 819 0 0 3812 0 0 4 0 33 0 0 33 0 0 0 0 0 0 0 0 0 + bytes: 3996123136 0 66593460224 0 70589710968 0 975648 0 + RPC iostats version: 1.1 p/v: 100003/3 (nfs) + xprt: tcp 918 1 2 0 1 118204 118204 0 281157 0 5 68 162921 + per-op statistics + NULL: 1 1 0 44 24 3 1 4 0 + GETATTR: 54 54 0 7744 6048 0 63 65 0 + SETATTR: 0 0 0 0 0 0 0 0 0 + LOOKUP: 19 19 0 3512 4484 0 19 19 0 + ACCESS: 9 9 0 1324 1080 0 9 9 0 + READLINK: 5 5 0 720 980 0 5 5 0 + READ: 118108 118108 0 18424848 70604828792 1928 178931 183807 0 + WRITE: 0 0 0 0 0 0 0 0 0 + CREATE: 0 0 0 0 0 0 0 0 0 + MKDIR: 0 0 0 0 0 0 0 0 0 + SYMLINK: 0 0 0 0 0 0 0 0 0 + MKNOD: 0 0 0 0 0 0 0 0 0 + REMOVE: 0 0 0 0 0 0 0 0 0 + RMDIR: 0 0 0 0 0 0 0 0 0 + RENAME: 0 0 0 0 0 0 0 0 0 + LINK: 0 0 0 0 0 0 0 0 0 + READDIR: 0 0 0 0 0 0 0 0 0 + READDIRPLUS: 2 2 0 336 3124 0 2 2 0 + FSSTAT: 2 2 0 272 168 0 2 2 0 + FSINFO: 2 2 0 272 160 0 2 2 0 + PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt b/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt new file mode 100644 index 00000000..bbfbe3a8 --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt @@ -0,0 +1,32 @@ +lrwxrwxrwx 1 root root 76 Sep 17 22:39 /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors -> ../../blobs/31d6a825ae35f11fb85b195b4c42c146c051e446433125a215336abdf95cbf5f +device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 + opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none + age: 40 + caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 + sec: flavor=1,pseudoflavor=1 + events: 2 102 0 0 2 11 125 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + bytes: 0 0 0 0 0 0 0 0 + RPC iostats version: 1.1 p/v: 100003/3 (nfs) + xprt: tcp 918 1 2 0 0 36 36 0 36 0 2 0 0 + per-op statistics + NULL: 1 1 0 44 24 3 1 4 0 + GETATTR: 4 4 0 552 448 0 6 6 0 + SETATTR: 0 0 0 0 0 0 0 0 0 + LOOKUP: 11 11 0 2008 2596 0 11 11 0 + ACCESS: 8 8 0 1176 960 0 8 8 0 + READLINK: 5 5 0 720 980 0 5 5 0 + READ: 0 0 0 0 0 0 0 0 0 + WRITE: 0 0 0 0 0 0 0 0 0 + CREATE: 0 0 0 0 0 0 0 0 0 + MKDIR: 0 0 0 0 0 0 0 0 0 + SYMLINK: 0 0 0 0 0 0 0 0 0 + MKNOD: 0 0 0 0 0 0 0 0 0 + REMOVE: 0 0 0 0 0 0 0 0 0 + RMDIR: 0 0 0 0 0 0 0 0 0 + RENAME: 0 0 0 0 0 0 0 0 0 + LINK: 0 0 0 0 0 0 0 0 0 + READDIR: 0 0 0 0 0 0 0 0 0 + READDIRPLUS: 2 2 0 336 3124 0 2 2 0 + FSSTAT: 1 1 0 136 84 0 0 0 0 + FSINFO: 2 2 0 272 160 0 2 2 0 + PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/summary.md b/runs/2026-10-15_cold_start/fio-l4-lvw8/summary.md new file mode 100644 index 00000000..c61a99eb --- /dev/null +++ b/runs/2026-10-15_cold_start/fio-l4-lvw8/summary.md @@ -0,0 +1,11 @@ +# fio from fio-l4-lvw8 + +file: file: /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors + +| job | MiB/s | IOPS | mean latency ms | read | seconds | +|---|---|---|---|---|---| +| fio-seqread-1M-direct | 478.7 | 479 | 2.09 | 3.72 GiB | 8.0 | +| fio-randread-4k-direct | 4.2 | 1066 | 0.94 | 0.12 GiB | 30.0 | +| fio-seqread-256k-qd1-direct | 210.2 | 841 | 1.19 | 6.16 GiB | 30.0 | +| fio-seqread-1M-4jobs-direct | 1775.4 | 1775 | 2.25 | 52.02 GiB | 30.0 | +| fio-seqread-1M-buffered | 893.3 | 893 | 1.12 | 3.72 GiB | 4.3 | diff --git a/runs/2026-10-15_cold_start/fix-image-tag.txt b/runs/2026-10-15_cold_start/fix-image-tag.txt new file mode 100644 index 00000000..a2dbcb0a --- /dev/null +++ b/runs/2026-10-15_cold_start/fix-image-tag.txt @@ -0,0 +1 @@ +TAG=coldstart1-dca985a diff --git a/runs/2026-10-15_cold_start/node-labels-before.json b/runs/2026-10-15_cold_start/node-labels-before.json new file mode 100644 index 00000000..ebfdfee8 --- /dev/null +++ b/runs/2026-10-15_cold_start/node-labels-before.json @@ -0,0 +1,42 @@ +{ + "gke-open-rl-dra-h100-sampler-a721f127-0853": { + "cloud.google.com/gke-nodepool": "h100-sampler", + "openrl.io/enabled": "true", + "openrl.io/sampler": "true" + }, + "gke-open-rl-dra-h100-sampler-a721f127-ed3t": { + "cloud.google.com/gke-nodepool": "h100-sampler", + "openrl.io/enabled": "true", + "openrl.io/sampler": "true" + }, + "gke-open-rl-dra-h100-trainer-702095b4-1g3a": { + "cloud.google.com/gke-nodepool": "h100-trainer", + "openrl.io/enabled": "true", + "openrl.io/trainer": "true" + }, + "gke-open-rl-dra-h100-trainer-702095b4-tuqu": { + "cloud.google.com/gke-nodepool": "h100-trainer", + "openrl.io/enabled": "true", + "openrl.io/trainer": "true" + }, + "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96": { + "cloud.google.com/gke-nodepool": "l4-sampler-2x", + "openrl.io/enabled": "true", + "openrl.io/sampler": "true" + }, + "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208": { + "cloud.google.com/gke-nodepool": "l4-sampler-2x", + "openrl.io/enabled": "true", + "openrl.io/sampler": "true" + }, + "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c": { + "cloud.google.com/gke-nodepool": "l4-trainer-2x", + "openrl.io/enabled": "true", + "openrl.io/trainer": "true" + }, + "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8": { + "cloud.google.com/gke-nodepool": "l4-trainer-2x", + "openrl.io/enabled": "true", + "openrl.io/trainer": "true" + } +} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/summary.md b/runs/2026-10-15_cold_start/summary.md new file mode 100644 index 00000000..f24c9f5b --- /dev/null +++ b/runs/2026-10-15_cold_start/summary.md @@ -0,0 +1,26 @@ +| run | total | create_model | trainer pull | trainer load | first sample | sampler pull | vLLM load | compile | capture | +|---|---|---|---|---|---|---|---|---|---| +| A-0.5b-l4-cold | 471.3 | 186.7 | 172.0 | 4.3 | 277.0 | 172.0 | 3.3 | 31.5 | 26.6 | +| B-0.5b-l4-cached | 132.0 | 17.6 | 4.0 | 4.2 | 106.7 | 4.0 | 3.0 | 31.4 | 26.7 | +| C-8b-h100-cold | 1350.2 | 1008.6 | 97.0 | 903.5 | 331.4 | 99.0 | 103.6 | 36.9 | 39.3 | +| D-8b-h100-cached | 182.6 | 52.2 | 4.0 | 39.9 | 120.7 | 3.0 | 13.5 | 37.4 | 39.5 | +| D-8b-h100-cached-interrupted | unmeasured | 555.2 | unmeasured | unmeasured | unmeasured | unmeasured | unmeasured | unmeasured | unmeasured | +| after-A1-0.5b-l4-cold | 493.7 | 197.1 | 172.0 | 15.0 | 289.2 | 186.0 | 2.4 | 31.5 | 26.7 | +| after-B1-0.5b-l4-cached | 157.4 | 15.5 | 3.0 | 2.5 | 134.6 | 3.0 | 2.5 | 31.4 | 26.9 | +| after-B2-0.5b-l4-cached | 130.5 | 15.8 | 4.0 | 2.5 | 107.4 | 4.0 | 2.6 | 31.3 | 26.8 | +| after-C1-8b-h100-cold | 329.3 | 111.5 | 90.0 | 13.0 | 207.9 | 90.0 | 14.0 | 37.5 | 39.9 | +| after-D1-8b-h100-cached | 154.5 | 25.3 | 3.0 | 13.5 | 119.4 | 4.0 | 12.9 | 37.6 | 39.3 | +| after-D2-8b-h100-cached | 154.3 | 24.1 | 4.0 | 12.1 | 119.9 | 4.0 | 13.1 | 37.2 | 39.5 | +| after-E1-gemma-e2b-h100-trainer-only | unmeasured | 103.7 | 3.0 | 92.5 | unmeasured | unmeasured | unmeasured | unmeasured | unmeasured | + +| group | n | total median (min-max) | create_model median (min-max) | trainer pull median (min-max) | trainer load median (min-max) | first sample median (min-max) | sampler pull median (min-max) | vLLM load median (min-max) | compile median (min-max) | capture median (min-max) | +|---|---|---|---|---|---|---|---|---|---|---| +| after A | 1 | 493.7 (493.7-493.7) | 197.1 (197.1-197.1) | 172.0 (172.0-172.0) | 15.0 (15.0-15.0) | 289.2 (289.2-289.2) | 186.0 (186.0-186.0) | 2.4 (2.4-2.4) | 31.5 (31.5-31.5) | 26.7 (26.7-26.7) | +| after B | 2 | 143.9 (130.5-157.4) | 15.7 (15.5-15.8) | 3.5 (3.0-4.0) | 2.5 (2.5-2.5) | 121.0 (107.4-134.6) | 3.5 (3.0-4.0) | 2.5 (2.5-2.6) | 31.4 (31.3-31.4) | 26.9 (26.8-26.9) | +| after C | 1 | 329.3 (329.3-329.3) | 111.5 (111.5-111.5) | 90.0 (90.0-90.0) | 13.0 (13.0-13.0) | 207.9 (207.9-207.9) | 90.0 (90.0-90.0) | 14.0 (14.0-14.0) | 37.5 (37.5-37.5) | 39.9 (39.9-39.9) | +| after D | 2 | 154.4 (154.3-154.5) | 24.7 (24.1-25.3) | 3.5 (3.0-4.0) | 12.8 (12.1-13.5) | 119.7 (119.4-119.9) | 4.0 (4.0-4.0) | 13.0 (12.9-13.1) | 37.4 (37.2-37.6) | 39.4 (39.3-39.5) | +| after E | 1 | unmeasured | 103.7 (103.7-103.7) | 3.0 (3.0-3.0) | 92.5 (92.5-92.5) | unmeasured | unmeasured | unmeasured | unmeasured | unmeasured | +| before A | 1 | 471.3 (471.3-471.3) | 186.7 (186.7-186.7) | 172.0 (172.0-172.0) | 4.3 (4.3-4.3) | 277.0 (277.0-277.0) | 172.0 (172.0-172.0) | 3.3 (3.3-3.3) | 31.5 (31.5-31.5) | 26.6 (26.6-26.6) | +| before B | 1 | 132.0 (132.0-132.0) | 17.6 (17.6-17.6) | 4.0 (4.0-4.0) | 4.2 (4.2-4.2) | 106.7 (106.7-106.7) | 4.0 (4.0-4.0) | 3.0 (3.0-3.0) | 31.4 (31.4-31.4) | 26.7 (26.7-26.7) | +| before C | 1 | 1350.2 (1350.2-1350.2) | 1008.6 (1008.6-1008.6) | 97.0 (97.0-97.0) | 903.5 (903.5-903.5) | 331.4 (331.4-331.4) | 99.0 (99.0-99.0) | 103.6 (103.6-103.6) | 36.9 (36.9-36.9) | 39.3 (39.3-39.3) | +| before D | 2 | 182.6 (182.6-182.6) | 303.7 (52.2-555.2) | 4.0 (4.0-4.0) | 39.9 (39.9-39.9) | 120.7 (120.7-120.7) | 3.0 (3.0-3.0) | 13.5 (13.5-13.5) | 37.4 (37.4-37.4) | 39.5 (39.5-39.5) | diff --git a/scripts/cold_start_fio.sh b/scripts/cold_start_fio.sh index bd407c98..2c3d40cf 100755 --- a/scripts/cold_start_fio.sh +++ b/scripts/cold_start_fio.sh @@ -47,13 +47,17 @@ echo "file: $FILE" | tee "$OUT/file.txt" kubectl -n "$NS" exec $POD -- bash -c "ls -l $FILE; grep -A30 ' /mnt/shared ' /proc/self/mountstats" > "$OUT/mountstats-before.txt" kubectl -n "$NS" exec $POD -- fio --name=seqread --filename="$FILE" --rw=read --bs=1M --direct=1 --ioengine=psync --numjobs=1 --runtime=60 --time_based=0 --output-format=json > "$OUT/fio-seqread-1M-direct.json" kubectl -n "$NS" exec $POD -- fio --name=randread --filename="$FILE" --rw=randread --bs=4k --direct=1 --ioengine=psync --numjobs=1 --runtime=30 --time_based=1 --output-format=json > "$OUT/fio-randread-4k-direct.json" +# The trainer's observed pattern: one synchronous ~256 KiB read at a time. +kubectl -n "$NS" exec $POD -- fio --name=seqread-256k-qd1 --filename="$FILE" --rw=read --bs=256k --direct=1 --ioengine=psync --numjobs=1 --runtime=30 --time_based=1 --output-format=json > "$OUT/fio-seqread-256k-qd1-direct.json" +# Headroom: four readers at once, 1 MiB each, different offsets of the same file. +kubectl -n "$NS" exec $POD -- fio --name=seqread-1M-4jobs --filename="$FILE" --rw=read --bs=1M --direct=1 --ioengine=psync --numjobs=4 --offset_increment=20% --runtime=30 --time_based=1 --group_reporting --output-format=json > "$OUT/fio-seqread-1M-4jobs-direct.json" kubectl -n "$NS" exec $POD -- fio --name=seqread-buffered --filename="$FILE" --rw=read --bs=1M --direct=0 --ioengine=psync --numjobs=1 --runtime=60 --time_based=0 --output-format=json > "$OUT/fio-seqread-1M-buffered.json" kubectl -n "$NS" exec $POD -- bash -c "grep -A30 ' /mnt/shared ' /proc/self/mountstats" > "$OUT/mountstats-after.txt" kubectl -n "$NS" delete pod "$POD" --wait=false >/dev/null python3 - "$OUT" <<'EOF' import json, sys, pathlib out = pathlib.Path(sys.argv[1]) -for name in ("fio-seqread-1M-direct", "fio-randread-4k-direct", "fio-seqread-1M-buffered"): +for name in ("fio-seqread-1M-direct", "fio-randread-4k-direct", "fio-seqread-256k-qd1-direct", "fio-seqread-1M-4jobs-direct", "fio-seqread-1M-buffered"): d = json.load(open(out / f"{name}.json"))["jobs"][0]["read"] print(f"{name:26} {d['bw_bytes']/2**20:8.1f} MiB/s {d['iops']:9.1f} IOPS mean lat {d['lat_ns']['mean']/1e6:7.2f} ms read {d['io_bytes']/2**30:.2f} GiB in {d['runtime']/1000:.1f} s") EOF diff --git a/scripts/cold_start_monitoring.py b/scripts/cold_start_monitoring.py new file mode 100644 index 00000000..b0e8b67c --- /dev/null +++ b/scripts/cold_start_monitoring.py @@ -0,0 +1,73 @@ +"""Filestore read throughput and IOPS from Cloud Monitoring over a time window. + + python3 scripts/cold_start_monitoring.py --start 2026-10-15T03:00:00Z --end 2026-10-15T03:10:00Z --out runs// + +Reads file.googleapis.com/nfs/server/read_bytes_count and read_ops_count for +every Filestore instance in the project, aligned to 60 s rates (the finest +Filestore publishes), and writes monitoring.json plus a per-minute table. +Uses application-default credentials (`gcloud auth application-default +print-access-token`); the plain user token is refused by this API. +""" + +import argparse +import json +import subprocess +import urllib.parse +from pathlib import Path + +PROJECT = "cdrollouts-sunilarora" +METRICS = { + "read_bytes_count": "file.googleapis.com/nfs/server/read_bytes_count", + "read_ops_count": "file.googleapis.com/nfs/server/read_ops_count", + "write_bytes_count": "file.googleapis.com/nfs/server/write_bytes_count", +} + + +def token() -> str: + return subprocess.run(["gcloud", "auth", "application-default", "print-access-token"], check=True, capture_output=True, text=True).stdout.strip() + + +def fetch(metric: str, start: str, end: str, tok: str) -> dict: + # curl rather than urllib: the system Python here has no CA bundle. + params = { + "filter": f'metric.type="{metric}"', + "interval.startTime": start, + "interval.endTime": end, + "aggregation.alignmentPeriod": "60s", + "aggregation.perSeriesAligner": "ALIGN_RATE", + } + url = f"https://monitoring.googleapis.com/v3/projects/{PROJECT}/timeSeries?" + urllib.parse.urlencode(params) + out = subprocess.run(["curl", "-sS", "-H", f"Authorization: Bearer {tok}", url], check=True, capture_output=True, text=True).stdout + return json.loads(out) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--start", required=True) + parser.add_argument("--end", required=True) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + tok = token() + result = {name: fetch(metric, args.start, args.end, tok) for name, metric in METRICS.items()} + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "monitoring.json").write_text(json.dumps(result, indent=1)) + per_minute: dict[str, dict[str, float]] = {} + for name, body in result.items(): + for series in body.get("timeSeries", []): + for point in series["points"]: + row = per_minute.setdefault(point["interval"]["endTime"], {}) + row[name] = float(point["value"].get("doubleValue", 0.0)) + lines = ["| minute ending (UTC) | read MiB/s | read ops/s | write MiB/s |", "|---|---|---|---|"] + for end_time in sorted(per_minute): + row = per_minute[end_time] + lines.append( + f"| {end_time} | {row.get('read_bytes_count', 0) / 2**20:.1f} | {row.get('read_ops_count', 0):.0f} " + f"| {row.get('write_bytes_count', 0) / 2**20:.1f} |" + ) + table = "\n".join(lines) + "\n" + (args.out / "monitoring.md").write_text(table) + print(table) + + +if __name__ == "__main__": + main() diff --git a/scripts/cold_start_resident.py b/scripts/cold_start_resident.py new file mode 100644 index 00000000..0f928bd8 --- /dev/null +++ b/scripts/cold_start_resident.py @@ -0,0 +1,52 @@ +"""How much of each checkpoint file is resident in this node's page cache. + + python3 scripts/cold_start_resident.py '/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/*/*.safetensors' + +Run inside a pod that mounts the shared volume (kubectl cp it in). mincore +answers per page without reading anything, so it does not disturb what it +measures. Used in runs/2026-10-15_cold_start to show that a pod's reads +survive in the page cache only while the volume stays mounted on the node. +""" + +import ctypes +import glob +import mmap +import os +import sys + +libc = ctypes.CDLL(None, use_errno=True) +PAGE = os.sysconf("SC_PAGE_SIZE") + + +def resident_pages(path: str) -> tuple[int, int]: + size = os.path.getsize(path) + fd = os.open(path, os.O_RDONLY) + try: + mm = mmap.mmap(fd, size, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ | mmap.PROT_WRITE) + try: + pages = (size + PAGE - 1) // PAGE + vec = (ctypes.c_ubyte * pages)() + addr = ctypes.addressof(ctypes.c_char.from_buffer(mm)) + if libc.mincore(ctypes.c_void_p(addr), ctypes.c_size_t(size), vec) != 0: + raise OSError(ctypes.get_errno(), "mincore failed") + resident = sum(1 for b in vec if b & 1) + del vec + return resident, pages + finally: + mm.close() + finally: + os.close(fd) + + +def main() -> None: + total_resident = total_pages = 0 + for path in sorted(glob.glob(sys.argv[1])): + resident, pages = resident_pages(path) + total_resident += resident + total_pages += pages + print(f"{os.path.basename(path):40} {pages * PAGE / 2**30:6.2f} GiB resident {100 * resident / max(pages, 1):5.1f}%") + print(f"TOTAL resident {100 * total_resident / max(total_pages, 1):.1f}% of {total_pages * PAGE / 2**30:.1f} GiB") + + +if __name__ == "__main__": + main() diff --git a/scripts/cold_start_summary.py b/scripts/cold_start_summary.py new file mode 100644 index 00000000..c364ba93 --- /dev/null +++ b/scripts/cold_start_summary.py @@ -0,0 +1,74 @@ +"""Summarize cold-start timelines across cases: one row per run, medians per group. + + python3 scripts/cold_start_summary.py runs/2026-10-15_cold_start + +Reads every /timeline.json the timeline tool produced and prints a +markdown table of the phases that matter for the demo, then medians and +ranges for groups of runs whose directory names share a stem after the +`after-` prefix and before a trailing run number (B1/B2 -> B). +""" + +import json +import re +import statistics +import sys +from datetime import datetime +from pathlib import Path + +PHASES = [ + ("total", "client.create_model.start", "client.sample.done"), + ("create_model", "client.create_model.start", "client.create_model.done"), + ("trainer pull", "trainer.pod_PodScheduled", "trainer.event_Pulled"), + ("trainer load", "trainer.load_start", "trainer.load_done"), + ("first sample", "client.sample.start", "client.sample.done"), + ("sampler pull", "sampler.pod_PodScheduled", "sampler.event_Pulled"), + ("vLLM load", "sampler.load_start", "sampler.weights_loaded"), + ("compile", "sampler.weights_loaded", "sampler.compile_done"), + ("capture", "sampler.compile_done", "sampler.engine_init_done"), +] + + +def ts(value): + if not value: + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def phases_of(report: dict) -> dict[str, float | None]: + marks = dict(report["client"]) + for w in report["workers"].values(): + marks.update(w["marks"]) + out = {} + for name, a, b in PHASES: + ta, tb = ts(marks.get(a, {}).get("ts")), ts(marks.get(b, {}).get("ts")) + out[name] = round((tb - ta).total_seconds(), 1) if ta and tb else None + return out + + +def main() -> None: + root = Path(sys.argv[1]) + rows = [] + for path in sorted(root.glob("*/timeline.json")): + report = json.loads(path.read_text()) + rows.append((path.parent.name, phases_of(report))) + names = [p[0] for p in PHASES] + print("| run | " + " | ".join(names) + " |") + print("|---|" + "---|" * len(names)) + for case, ph in rows: + print(f"| {case} | " + " | ".join("unmeasured" if ph[n] is None else f"{ph[n]:.1f}" for n in names) + " |") + groups: dict[str, list[dict]] = {} + for case, ph in rows: + stem = re.sub(r"^(after-)?([A-Z])\d*-.*$", lambda m: ("after " if m.group(1) else "before ") + m.group(2), case) + groups.setdefault(stem, []).append(ph) + print("\n| group | n | " + " | ".join(f"{n} median (min-max)" for n in names) + " |") + print("|---|---|" + "---|" * len(names)) + for stem, phs in sorted(groups.items()): + cells = [] + for n in names: + vals = [p[n] for p in phs if p[n] is not None] + cells.append("unmeasured" if not vals else f"{statistics.median(vals):.1f} ({min(vals):.1f}-{max(vals):.1f})") + print(f"| {stem} | {len(phs)} | " + " | ".join(cells) + " |") + + +if __name__ == "__main__": + main() diff --git a/scripts/cold_start_timeline.py b/scripts/cold_start_timeline.py index 3e9123ab..9f19b8bd 100644 --- a/scripts/cold_start_timeline.py +++ b/scripts/cold_start_timeline.py @@ -118,15 +118,23 @@ def cmd_watch(out: Path) -> None: poll_loop(["kubectl", "-n", NS, "get", "pods", "-o", f"jsonpath={POD_JSONPATH}"], out / "pods.tsv"), stamped_follower( [ - "kubectl", "-n", NS, "exec", "deploy/redis-store", "--", "sh", "-c", - 'while true; do t=$(date -u +%Y-%m-%dT%H:%M:%SZ); for k in $(redis-cli --scan --pattern "open_rl:*_ready:*"); do echo "$t $k"; done; sleep 1; done', + "kubectl", + "-n", + NS, + "exec", + "deploy/redis-store", + "--", + "sh", + "-c", + "while true; do t=$(date -u +%Y-%m-%dT%H:%M:%SZ); " + 'for k in $(redis-cli --scan --pattern "open_rl:*_ready:*"); do echo "$t $k"; done; sleep 1; done', ], out / "redis-ready.log", ), stamped_follower(["kubectl", "-n", NS, "logs", "-f", "--timestamps", "--tail=0", "deploy/open-rl-api-server"], out / "api-server.log"), stamped_follower(["kubectl", "-n", NS, "logs", "-f", "--timestamps", "--tail=0", "deploy/open-rl-scheduler"], out / "scheduler.log"), ] - pids = [p.pid for p in procs] + [getattr(p, "producer").pid for p in procs if hasattr(p, "producer")] + pids = [p.pid for p in procs] + [p.producer.pid for p in procs if hasattr(p, "producer")] # type: ignore[attr-defined] (out / "watch.pids").write_text("\n".join(map(str, pids)) + "\n") print(f"watching; recorders {pids} write into {out}") @@ -194,6 +202,9 @@ def duration_s(text: str) -> float | None: SAMPLER_MARKS = [ ("sampler.process_first_log", None), + ("sampler.engine_init_start", r"Initializing a V1 LLM engine"), + ("sampler.load_start", r"Loading model from scratch"), + ("sampler.prefetch_start", r"Prefetching checkpoint files into page cache started"), ("sampler.weights_loaded", r"Loading weights took ([\d.]+) seconds"), ("sampler.model_loaded", r"Model loading took .* and ([\d.]+) seconds"), ("sampler.compile_done", r"torch\.compile took ([\d.]+) s"), @@ -266,20 +277,28 @@ def ready_keys(path: Path) -> dict[str, datetime]: return seen -def pod_events(events: dict[str, Any], pod: str) -> list[dict[str, Any]]: +def pod_events(events: dict[str, Any], pod: dict[str, Any]) -> list[dict[str, Any]]: + """Events for this pod object, not for an earlier pod of the same name: the + workload names repeat across runs, so match the pod's uid.""" + name, uid = pod["metadata"]["name"], pod["metadata"].get("uid") rows = [] for e in events.get("items", []): - if e["involvedObject"].get("name") != pod: + obj = e["involvedObject"] + if obj.get("name") != name or (uid and obj.get("uid") and obj["uid"] != uid): continue rows.append({"reason": e["reason"], "ts": parse_ts(e.get("firstTimestamp") or e.get("eventTime")), "message": e.get("message", "")}) - return sorted(rows, key=lambda r: (r["ts"] or datetime.min.replace(tzinfo=UTC))) + return sorted(rows, key=lambda r: r["ts"] or datetime.min.replace(tzinfo=UTC)) def cmd_timeline(out: Path) -> None: client = [json.loads(l) for l in (out / "client.jsonl").read_text().splitlines() if l.strip()] if (out / "client.jsonl").exists() else [] marks: dict[str, dict[str, Any]] = {} for row in client: - marks[f"client.{row['phase']}"] = {"ts": parse_ts(row["ts"]), "source": "client.jsonl (laptop clock)", **{k: v for k, v in row.items() if k not in ("ts", "t", "phase")}} + marks[f"client.{row['phase']}"] = { + "ts": parse_ts(row["ts"]), + "source": "client.jsonl (laptop clock)", + **{k: v for k, v in row.items() if k not in ("ts", "t", "phase")}, + } pods = json.loads((out / "pods.json").read_text()) if (out / "pods.json").exists() else {"items": []} events = json.loads((out / "events.json").read_text()) if (out / "events.json").exists() else {"items": []} @@ -293,7 +312,13 @@ def cmd_timeline(out: Path) -> None: for wl in workloads.get("items", []): role = wl["spec"]["role"] name = wl["metadata"]["name"] - w: dict[str, Any] = {"workload": name, "model_id": wl["spec"].get("modelID"), "node": wl.get("status", {}).get("nodeName"), "pod": wl.get("status", {}).get("podName"), "marks": {}} + w: dict[str, Any] = { + "workload": name, + "model_id": wl["spec"].get("modelID"), + "node": wl.get("status", {}).get("nodeName"), + "pod": wl.get("status", {}).get("podName"), + "marks": {}, + } w["marks"][f"{role}.workload_created"] = {"ts": parse_ts(wl["metadata"]["creationTimestamp"]), "source": "Workload creationTimestamp (apiserver)"} if name in wl_first_seen: w["marks"][f"{role}.workload_first_seen_by_poller"] = {"ts": wl_first_seen[name], "source": "workloads.tsv poller (laptop clock, 1 s)"} @@ -302,7 +327,10 @@ def cmd_timeline(out: Path) -> None: if name in wl_first_node: w["marks"][f"{role}.workload_reports_node"] = {"ts": wl_first_node[name], "source": "workloads.tsv poller: status.nodeName set"} for cond in wl.get("status", {}).get("conditions", []): - w["marks"][f"{role}.condition_{cond['type']}_{cond['status']}"] = {"ts": parse_ts(cond.get("lastTransitionTime")), "source": f"Workload condition {cond.get('reason', '')}"} + w["marks"][f"{role}.condition_{cond['type']}_{cond['status']}"] = { + "ts": parse_ts(cond.get("lastTransitionTime")), + "source": f"Workload condition {cond.get('reason', '')}", + } workers[role] = w for pod in pods.get("items", []): @@ -322,7 +350,7 @@ def cmd_timeline(out: Path) -> None: if running.get("startedAt"): w["marks"][f"{role}.container_started"] = {"ts": parse_ts(running["startedAt"]), "source": "containerStatuses.state.running.startedAt"} w["image_id"] = cs.get("imageID") - for ev in pod_events(events, name): + for ev in pod_events(events, pod): key = f"{role}.event_{ev['reason']}" if key in w["marks"]: continue @@ -333,7 +361,6 @@ def cmd_timeline(out: Path) -> None: w["marks"][key] = entry log = out / f"log-{name}.txt" w["marks"].update(scan_log(log, SAMPLER_MARKS if role == "sampler" else TRAINER_MARKS)) - model_id = w.get("model_id") or (workers.get("trainer", {}).get("model_id")) for key, when in ready.items(): if key.startswith(f"open_rl:{role}_ready:"): w["marks"][f"{role}.ready_key"] = {"ts": when, "source": f"redis poller: {key}"} @@ -353,14 +380,22 @@ def render(report: dict[str, Any]) -> str: t0 = client.get("client.create_model.start", {}).get("ts") lines = [f"# Cold start timeline: {report['case']}", ""] if t0: - lines.append(f"t=0 is the client's `create_model` call at {t0.isoformat(timespec='milliseconds')} UTC. Offsets are seconds after t=0; a negative offset is before it.") + lines.append( + f"t=0 is the client's `create_model` call at {t0.isoformat(timespec='milliseconds')} UTC. " + "Offsets are seconds after t=0; a negative offset is before it." + ) lines.append("") for role in ("trainer", "sampler"): w = report["workers"].get(role) if not w: lines += [f"## {role}", "", "no worker observed", ""] continue - lines += [f"## {role}: `{w.get('pod')}` on `{w.get('node')}`", "", "| Mark | Clock (UTC) | t+ s | Self-reported s | Source |", "|---|---|---|---|---|"] + lines += [ + f"## {role}: `{w.get('pod')}` on `{w.get('node')}`", + "", + "| Mark | Clock (UTC) | t+ s | Self-reported s | Source |", + "|---|---|---|---|---|", + ] rows = sorted(w["marks"].items(), key=lambda kv: (kv[1].get("ts") is None, kv[1].get("ts") or datetime.max.replace(tzinfo=UTC))) for name, m in rows: ts = m.get("ts") @@ -398,7 +433,9 @@ def critical_path(report: dict[str, Any]) -> list[str]: ("sampler.pod_PodScheduled", "sampler.event_Pulled", "image pull (0 if cached)"), ("sampler.event_Pulled", "sampler.container_started", "container create"), ("sampler.container_started", "sampler.process_first_log", "process start to first log"), - ("sampler.process_first_log", "sampler.weights_loaded", "imports and engine setup until weights read"), + ("sampler.process_first_log", "sampler.engine_init_start", "imports until engine init"), + ("sampler.engine_init_start", "sampler.load_start", "engine init until load start"), + ("sampler.load_start", "sampler.weights_loaded", "weights load (vLLM prefetch into page cache + load)"), ("sampler.weights_loaded", "sampler.compile_done", "torch.compile"), ("sampler.compile_done", "sampler.engine_init_done", "profiling, KV cache, graph capture"), ("sampler.engine_init_done", "sampler.ready_key", "ready key"), From 231c9348a638c74cc929ac451db92405a43744a3 Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 09:16:22 -0700 Subject: [PATCH 03/19] scripts: render and report N in-cluster GSM8K GRPO client Jobs for the fleet demo --- scripts/demo_jobs.py | 123 +++++++++++++++++++++++++++++++++++++++++ scripts/demo_report.py | 92 ++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 scripts/demo_jobs.py create mode 100644 scripts/demo_report.py diff --git a/scripts/demo_jobs.py b/scripts/demo_jobs.py new file mode 100644 index 00000000..818f7175 --- /dev/null +++ b/scripts/demo_jobs.py @@ -0,0 +1,123 @@ +"""Render N in-cluster RL client Jobs for the fleet demo, from a short spec. + + python3 scripts/demo_jobs.py --image --run demo-01 \\ + --job lora:Qwen/Qwen3-0.6B:3 --job fft:google/gemma-4-e2b:2 --steps 3 | kubectl apply -f - + +Each --job is kind:model:count. kind is lora or fft. Every Job runs the GSM8K +GRPO recipe (scripts/run_training_e2e.py, scenario lora-gsm8k-rl or +fft-gsm8k-rl) against the API server's in-cluster Service, so no port-forward +is involved. Jobs are named --- and labeled +openrl.io/demo-run=, so one selector lists, watches or deletes them all. +Each Job writes its recipe output under +/mnt/shared/open-rl/runs/demo///, where scripts/demo_report.py +reads the step timings. +""" + +import argparse +import re +import sys + +TEMPLATE = """--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {name} + namespace: {namespace} + labels: + app: openrl-demo + openrl.io/demo-run: "{run}" + openrl.io/demo-kind: "{kind}" + openrl.io/demo-model: "{slug}" +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 172800 + template: + metadata: + labels: + app: openrl-demo + openrl.io/demo-run: "{run}" + spec: + restartPolicy: Never + serviceAccountName: open-rl-sa + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: client + image: {image} + imagePullPolicy: IfNotPresent + args: + - "scenario={scenario}" + - "uv_extra=cpu" + - "base_url=http://open-rl-api-server-service:8000" + - "base_model={model}" + - "steps={steps}" + - "group_size={group_size}" + - "groups_per_batch={groups_per_batch}" + - "max_tokens={max_tokens}" + - "log_dir=/mnt/shared/open-rl/runs/demo/{run}/{name}" + env: + - name: PYTHONUNBUFFERED + value: "1" + - name: HF_HOME + value: "/mnt/shared/open-rl/huggingface" + resources: + requests: + cpu: "1" + memory: "4Gi" + limits: + memory: "8Gi" + volumeMounts: + - name: shared-storage + mountPath: /mnt/shared + volumes: + - name: shared-storage + persistentVolumeClaim: + claimName: open-rl-shared-pvc +""" + + +def slug(model: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", model.lower()).strip("-")[:30] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--image", required=True) + parser.add_argument("--run", required=True, help="name of this demo run; also the label value") + parser.add_argument("--job", action="append", required=True, help="kind:model:count, kind in {lora, fft}") + parser.add_argument("--steps", type=int, default=3) + parser.add_argument("--group-size", type=int, default=8) + parser.add_argument("--groups-per-batch", type=int, default=8) + parser.add_argument("--max-tokens", type=int, default=512) + parser.add_argument("--namespace", default="openrl-system") + args = parser.parse_args() + out = [] + for spec in args.job: + kind, model, count = spec.rsplit(":", 2) + if kind not in ("lora", "fft"): + sys.exit(f"kind must be lora or fft, got {kind!r}") + for n in range(1, int(count) + 1): + name = f"{args.run}-{kind}-{slug(model)}-{n:02d}" + out.append( + TEMPLATE.format( + name=name, + namespace=args.namespace, + run=args.run, + kind=kind, + slug=slug(model), + image=args.image, + scenario=f"{kind}-gsm8k-rl", + model=model, + steps=args.steps, + group_size=args.group_size, + groups_per_batch=args.groups_per_batch, + max_tokens=args.max_tokens, + ) + ) + sys.stdout.write("".join(out)) + + +if __name__ == "__main__": + main() diff --git a/scripts/demo_report.py b/scripts/demo_report.py new file mode 100644 index 00000000..7d943006 --- /dev/null +++ b/scripts/demo_report.py @@ -0,0 +1,92 @@ +"""Per-job step timings for a demo run, from the Jobs and the recipes' metrics. + + python3 scripts/demo_report.py --run demo-01 [--namespace openrl-system] + +For every Job labeled openrl.io/demo-run=: when the pod started, when the +first and last step finished, and the recipe's own per-step times +(time/sampling, time/train_step, time/total) from metrics.jsonl on the shared +volume, read through the API server pod. Prints one row per job and the share +of jobs that finished N steps within a budget. +""" + +import argparse +import json +import subprocess +from datetime import datetime + + +def sh(*args: str) -> str: + return subprocess.run(args, check=True, capture_output=True, text=True).stdout + + +def ts(value: str | None) -> datetime | None: + return datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run", required=True) + parser.add_argument("--namespace", default="openrl-system") + parser.add_argument("--budget", type=float, default=180.0, help="seconds from pod start for the finish-rate line") + parser.add_argument("--steps", type=int, default=2, help="steps that count as finished for the finish-rate line") + args = parser.parse_args() + ns = args.namespace + jobs = json.loads(sh("kubectl", "-n", ns, "get", "jobs", "-l", f"openrl.io/demo-run={args.run}", "-o", "json"))["items"] + pods = json.loads(sh("kubectl", "-n", ns, "get", "pods", "-l", f"openrl.io/demo-run={args.run}", "-o", "json"))["items"] + pod_by_job = {p["metadata"]["labels"].get("job-name"): p for p in pods} + # One cat per job through the API server pod, which mounts the volume. + names = [j["metadata"]["name"] for j in jobs] + script = ( + 'for n in %s; do for f in /mnt/shared/open-rl/runs/demo/%s/$n/open-rl-tmp/*_gsm8k_rl/metrics.jsonl; do [ -f "$f" ] && { echo "== $n"; cat "$f"; }; done; done' + % ( + " ".join(names), + args.run, + ) + ) + raw = sh("kubectl", "-n", ns, "exec", "deploy/open-rl-api-server", "--", "sh", "-c", script) if names else "" + metrics: dict[str, list[dict]] = {} + current = None + for line in raw.splitlines(): + if line.startswith("== "): + current = line[3:].strip() + metrics[current] = [] + elif current and line.strip(): + try: + metrics[current].append(json.loads(line)) + except ValueError: + pass + print(f"| job | pod start | status | steps | first step at +s | step {args.steps} at +s | sampling s/step | train s/step | total s/step |") + print("|---|---|---|---|---|---|---|---|---|") + finished = 0 + for job in sorted(jobs, key=lambda j: j["metadata"]["name"]): + name = job["metadata"]["name"] + pod = pod_by_job.get(name, {}) + start = ts(pod.get("status", {}).get("startTime")) + status = ( + "succeeded" if job["status"].get("succeeded") else "failed" if job["status"].get("failed") else pod.get("status", {}).get("phase", "?").lower() + ) + rows = [r for r in metrics.get(name, []) if "time/total" in r] + first = step_n = None + if rows and start: + # metrics rows carry no wall clock; derive from the pod start plus cumulative time/total, + # which the recipe measures per step including startup of the first. + elapsed = 0.0 + for i, r in enumerate(rows, 1): + elapsed += float(r.get("time/total", 0)) + if i == 1: + first = elapsed + if i == args.steps: + step_n = elapsed + if step_n is not None and step_n <= args.budget: + finished += 1 + mean = lambda key: f"{sum(float(r.get(key, 0)) for r in rows) / len(rows):.1f}" if rows else "" + print( + f"| {name} | {start.strftime('%H:%M:%S') if start else ''} | {status} | {len(rows)} | {'' if first is None else f'{first:.0f}'} | " + f"{'' if step_n is None else f'{step_n:.0f}'} | {mean('time/sampling')} | {mean('time/train_step')} | {mean('time/total')} |" + ) + if jobs: + print(f"\n{finished}/{len(jobs)} jobs reached step {args.steps} within {args.budget:.0f} s of pod start ({100 * finished / len(jobs):.0f}%).") + + +if __name__ == "__main__": + main() From 65350ba5512f814cde1b48c48349f77708c7145e Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 09:30:08 -0700 Subject: [PATCH 04/19] scripts: per-step rows in the demo report --- scripts/demo_report.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/demo_report.py b/scripts/demo_report.py index 7d943006..4f4c9a05 100644 --- a/scripts/demo_report.py +++ b/scripts/demo_report.py @@ -29,6 +29,7 @@ def main() -> None: parser.add_argument("--namespace", default="openrl-system") parser.add_argument("--budget", type=float, default=180.0, help="seconds from pod start for the finish-rate line") parser.add_argument("--steps", type=int, default=2, help="steps that count as finished for the finish-rate line") + parser.add_argument("--per-step", action="store_true", help="also print every step of every job") args = parser.parse_args() ns = args.namespace jobs = json.loads(sh("kubectl", "-n", ns, "get", "jobs", "-l", f"openrl.io/demo-run={args.run}", "-o", "json"))["items"] @@ -86,6 +87,15 @@ def main() -> None: ) if jobs: print(f"\n{finished}/{len(jobs)} jobs reached step {args.steps} within {args.budget:.0f} s of pod start ({100 * finished / len(jobs):.0f}%).") + if args.per_step: + print("\n| job | step | sampling s | train s | save s | total s |") + print("|---|---|---|---|---|---|") + for name in sorted(metrics): + for i, r in enumerate([r for r in metrics[name] if "time/total" in r], 1): + print( + f"| {name} | {i} | {float(r.get('time/sampling', 0)):.1f} | {float(r.get('time/train_step', 0)):.1f} | " + f"{float(r.get('time/save_checkpoint', 0)):.1f} | {float(r.get('time/total', 0)):.1f} |" + ) if __name__ == "__main__": From 22fa006d1485e8f484f011b69f2d167b0ae52d5c Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 12:30:07 -0700 Subject: [PATCH 05/19] trainer: size the per-pass token budget from the vocabulary and free device memory The cluster deployments never set OPEN_RL_TRAIN_TOKEN_BUDGET, so every forward_backward ran one sequence per pass: 64 passes for a 64-rollout GRPO step, 20 s of training per step for a 0.6B model on an H100. One fleet-wide number does not work either: 65536 tokens of gemma-4-e2b logits (262k vocabulary) is 22 GiB and OOMs an H100. The trainer now computes the budget from the model's vocab_size and the free memory on its device, and caps an explicit setting the same way. Measured: 0.6B training 20 s -> 4 s, gemma 31-35 s -> 8-10 s per step on H100 (runs/2026-10-17_demo_baselines). --- src/training/automodel_worker.py | 4 +-- src/training/trainer_worker.py | 46 +++++++++++++++++++++++++++++--- tests/test_prefetch.py | 30 +++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/src/training/automodel_worker.py b/src/training/automodel_worker.py index abaf6e22..645e6229 100644 --- a/src/training/automodel_worker.py +++ b/src/training/automodel_worker.py @@ -412,11 +412,11 @@ def close_cp_context(self) -> None: self.cp_context.close() self.cp_context = None - def make_training_batches(self, data: list[Datum]) -> list[list[tuple[int, Datum]]]: + def make_training_batches(self, data: list[Datum], token_budget: int | None = None) -> list[list[tuple[int, Datum]]]: # The CP shard takes one unpadded sequence per pass. if self.cp_size > 1: return [[(idx, datum)] for idx, datum in enumerate(data)] - return super().make_training_batches(data) + return super().make_training_batches(data, token_budget) def compute_target_logprobs( self, model: torch.nn.Module, input_ids: torch.Tensor, attention_mask: torch.Tensor, target_token_ids: torch.Tensor diff --git a/src/training/trainer_worker.py b/src/training/trainer_worker.py index 0178cde1..053d69b6 100644 --- a/src/training/trainer_worker.py +++ b/src/training/trainer_worker.py @@ -107,7 +107,7 @@ def _run_batches( shard_count = self.shard_count() local_indices = shard_datum_indices(len(data), self.shard_rank(), shard_count) local_data = [data[idx] for idx in local_indices] - local_batches = self.make_training_batches(local_data) + local_batches = self.make_training_batches(local_data, self.train_token_budget(model)) if shard_count > 1 and self.backward_runs_collectives: filler_passes = self.shard_all_reduce_max(len(local_batches)) - len(local_batches) filler = local_data[0] if local_data else data[0] @@ -184,12 +184,50 @@ def _finish(self, data: list[Datum], loss_fn_outputs: list[dict[str, Any] | None "loss_fn_output_type": "ArrayRecord", } - def make_training_batches(self, data: list[Datum]) -> list[list[tuple[int, Datum]]]: - """Group examples for the single padded forward/backward path.""" + def train_token_budget(self, model: PreTrainedModel) -> int: + """Padded tokens per forward/backward pass. + + Unset or `auto`: sized from the model's vocabulary and the free device + memory, because the logits are what a batch costs. A step of 64 sequences + then runs as a few passes instead of 64 (measured on an H100: 20 s to 4 s + for a 0.6B model). A number is used as given, but still capped by the + same bound, because one number cannot fit every model: 65536 tokens of + gemma-4-e2b logits (262k vocabulary) is 22 GiB and does not fit an H100. + `0` keeps one sequence per pass. + """ + raw = os.getenv("OPEN_RL_TRAIN_TOKEN_BUDGET", "auto").strip().lower() + if raw in ("0", "off"): + return 0 + cap = self.logits_token_cap(model) + if raw == "auto" or not raw: + return cap + return min(int(raw), cap) if cap else int(raw) + + # Bytes a token costs at the output layer: logits in the model dtype, the + # log-softmax, and what autograd keeps for backward. Half the free memory + # is left for activations and the optimizer. + LOGITS_BYTES_PER_VOCAB_ENTRY = 6 + LOGITS_MEMORY_SHARE = 0.5 + DEFAULT_TOKEN_CAP = 65536 + + def logits_token_cap(self, model: PreTrainedModel) -> int: + config = getattr(model, "config", None) + text_config = config.get_text_config() if config is not None and hasattr(config, "get_text_config") else config + vocab = int(getattr(text_config, "vocab_size", 0) or 0) + if vocab <= 0 or self.device.type != "cuda": + return self.DEFAULT_TOKEN_CAP + free_bytes, _ = torch.cuda.mem_get_info(self.device) + return max(1024, int(free_bytes * self.LOGITS_MEMORY_SHARE / (vocab * self.LOGITS_BYTES_PER_VOCAB_ENTRY))) + + def make_training_batches(self, data: list[Datum], token_budget: int | None = None) -> list[list[tuple[int, Datum]]]: + """Group examples for the single padded forward/backward path. Without an + explicit budget the environment's number applies, 0 meaning one per pass.""" if len(data) <= 1: return [[(idx, datum)] for idx, datum in enumerate(data)] - token_budget = int(os.getenv("OPEN_RL_TRAIN_TOKEN_BUDGET", "0")) + if token_budget is None: + raw = os.getenv("OPEN_RL_TRAIN_TOKEN_BUDGET", "0").strip().lower() + token_budget = 0 if raw in ("auto", "off", "") else int(raw) if token_budget <= 0: return [[(idx, datum)] for idx, datum in enumerate(data)] diff --git a/tests/test_prefetch.py b/tests/test_prefetch.py index a3df9ec1..3daf96fe 100644 --- a/tests/test_prefetch.py +++ b/tests/test_prefetch.py @@ -64,3 +64,33 @@ def test_large_files_are_split_across_threads(self): if __name__ == "__main__": unittest.main() + + +class TrainTokenBudgetTest(unittest.TestCase): + """The per-pass token budget follows the vocabulary and the free memory unless pinned.""" + + def setUp(self): + from training.trainer_worker import BaseTrainerWorker + + self.worker = BaseTrainerWorker() + self.model = type("Model", (), {"config": type("Config", (), {"vocab_size": 262144})()})() + + def test_auto_is_the_logits_cap(self): + with patch.dict(os.environ, {"OPEN_RL_TRAIN_TOKEN_BUDGET": "auto"}), patch.object(type(self.worker), "logits_token_cap", return_value=7000): + self.assertEqual(self.worker.train_token_budget(self.model), 7000) + with patch.dict(os.environ, {}, clear=False), patch.object(type(self.worker), "logits_token_cap", return_value=7000): + os.environ.pop("OPEN_RL_TRAIN_TOKEN_BUDGET", None) + self.assertEqual(self.worker.train_token_budget(self.model), 7000) + + def test_a_number_is_capped_by_what_the_logits_fit(self): + with patch.dict(os.environ, {"OPEN_RL_TRAIN_TOKEN_BUDGET": "65536"}), patch.object(type(self.worker), "logits_token_cap", return_value=7000): + self.assertEqual(self.worker.train_token_budget(self.model), 7000) + with patch.dict(os.environ, {"OPEN_RL_TRAIN_TOKEN_BUDGET": "4096"}), patch.object(type(self.worker), "logits_token_cap", return_value=7000): + self.assertEqual(self.worker.train_token_budget(self.model), 4096) + + def test_zero_keeps_one_sequence_per_pass(self): + with patch.dict(os.environ, {"OPEN_RL_TRAIN_TOKEN_BUDGET": "0"}): + self.assertEqual(self.worker.train_token_budget(self.model), 0) + + def test_cap_without_cuda_or_vocabulary_is_the_default(self): + self.assertEqual(self.worker.logits_token_cap(object()), self.worker.DEFAULT_TOKEN_CAP) From 6ff6b6f6a691376bfc57323dce915ed83c28949b Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 13:10:53 -0700 Subject: [PATCH 06/19] trainer: log the token budget and pass count once --- src/training/trainer_worker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/training/trainer_worker.py b/src/training/trainer_worker.py index 053d69b6..46a280dc 100644 --- a/src/training/trainer_worker.py +++ b/src/training/trainer_worker.py @@ -30,6 +30,7 @@ def __init__(self): # When the model load started and finished, and what the page-cache # prefetch did; the request processor publishes it with the ready key. self.load_phases: dict[str, Any] = {} + self.token_budget_logged = False if torch.cuda.is_available(): self.device = torch.device("cuda", local_rank()) @@ -107,7 +108,11 @@ def _run_batches( shard_count = self.shard_count() local_indices = shard_datum_indices(len(data), self.shard_rank(), shard_count) local_data = [data[idx] for idx in local_indices] - local_batches = self.make_training_batches(local_data, self.train_token_budget(model)) + token_budget = self.train_token_budget(model) + local_batches = self.make_training_batches(local_data, token_budget) + if not self.token_budget_logged: + print(f"[trainer] token budget per pass: {token_budget or 'one sequence'}; {len(local_data)} sequences in {len(local_batches)} pass(es)") + self.token_budget_logged = True if shard_count > 1 and self.backward_runs_collectives: filler_passes = self.shard_all_reduce_max(len(local_batches)) - len(local_batches) filler = local_data[0] if local_data else data[0] From 3f5d1bb6377e021312f755d79ebf69c964231c2b Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 13:36:49 -0700 Subject: [PATCH 07/19] deploy: pre-pull the worker image on every node; keep vLLM's compile cache on the node A worker pull is 92 s on an H100 node and 169 s on an L4 node, paid twice by a cold job (runs/2026-10-15_cold_start). The open-rl-worker-prepull DaemonSet in the base bundle pulls the image at deploy time on every openrl.io/enabled node, and workers now default to imagePullPolicy IfNotPresent so the pulled image is what runs. vLLM compiles the model on every sampler start, 28-37 s on an H100 and 122 s on an L4 for gemma-4-e2b, and kept the result in the container. Samplers now mount a hostPath (/var/lib/openrl/vllm-cache, OPEN_RL_VLLM_CACHE_HOSTPATH) as VLLM_CACHE_ROOT, so the second sampler of a model on a node skips it. --- k8s/deploy/base/kustomization.yaml | 1 + k8s/deploy/base/worker-prepull.yaml | 44 ++++++++++++++++++++++++++ src/server/scheduler_worker_manager.py | 25 +++++++++++++-- tests/test_scheduler_worker_manager.py | 31 ++++++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 k8s/deploy/base/worker-prepull.yaml diff --git a/k8s/deploy/base/kustomization.yaml b/k8s/deploy/base/kustomization.yaml index 1c95858c..ef4e9924 100644 --- a/k8s/deploy/base/kustomization.yaml +++ b/k8s/deploy/base/kustomization.yaml @@ -9,6 +9,7 @@ resources: - rbac.yaml - redis.yaml - api-server.yaml + - worker-prepull.yaml generatorOptions: disableNameSuffixHash: true diff --git a/k8s/deploy/base/worker-prepull.yaml b/k8s/deploy/base/worker-prepull.yaml new file mode 100644 index 00000000..41a70cd3 --- /dev/null +++ b/k8s/deploy/base/worker-prepull.yaml @@ -0,0 +1,44 @@ +# Pulls the worker image onto every node the scheduler may use, at deploy time +# instead of inside the first job. Measured on GKE (runs/2026-10-15_cold_start): +# a worker pull is 92 s on an H100 node with local SSD and 169 s on an L4 node +# with a pd-balanced boot disk, and a cold job pays it twice, trainer and +# sampler. The container only sleeps; it holds no GPU. The image reference is +# rewritten by the kustomize images transformer like every other OpenRL image, +# so a tag bump in the bundle re-pulls on every node as the DaemonSet rolls. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: open-rl-worker-prepull + labels: + app: open-rl-worker-prepull +spec: + selector: + matchLabels: + app: open-rl-worker-prepull + updateStrategy: + type: RollingUpdate + rollingUpdate: + # Every node pulls at once; there is nothing to protect by rolling slowly. + maxUnavailable: 100% + template: + metadata: + labels: + app: open-rl-worker-prepull + spec: + nodeSelector: + openrl.io/enabled: "true" + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: prepull + image: ghcr.io/gke-labs/open-rl/server:latest + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] + resources: + requests: + cpu: "10m" + memory: "16Mi" + limits: + memory: "64Mi" diff --git a/src/server/scheduler_worker_manager.py b/src/server/scheduler_worker_manager.py index 29743bed..c0b59ea6 100644 --- a/src/server/scheduler_worker_manager.py +++ b/src/server/scheduler_worker_manager.py @@ -126,6 +126,8 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: # No other worker shares an exclusive worker's GPUs, so it never parks. if worker.exclusive: values["OPEN_RL_TIME_SLICING"] = "off" + if worker.role == "sampler" and vllm_cache_hostpath(): + values["VLLM_CACHE_ROOT"] = VLLM_CACHE_MOUNT env: list[dict[str, Any]] = [{"name": name, "value": value} for name, value in values.items()] env.append({"name": "OPEN_RL_ACCEL_TIMESLICER_HOST", "valueFrom": {"fieldRef": {"fieldPath": "status.hostIP"}}}) return env @@ -177,11 +179,30 @@ def pod_template(worker: Worker) -> dict[str, Any]: template["spec"]["containers"][0]["volumeMounts"].append({"name": "dshm", "mountPath": "/dev/shm"}) template["spec"]["volumes"].append({"name": "dshm", "emptyDir": {"medium": "Memory"}}) - if pull_policy := os.getenv("OPEN_RL_WORKER_IMAGE_PULL_POLICY"): - template["spec"]["containers"][0]["imagePullPolicy"] = pull_policy + # Explicit, because Kubernetes defaults a :latest tag to Always and every + # worker start would then re-pull. The prepull DaemonSet puts the image on + # the node ahead of time; IfNotPresent is what makes that count. + template["spec"]["containers"][0]["imagePullPolicy"] = os.getenv("OPEN_RL_WORKER_IMAGE_PULL_POLICY", "IfNotPresent") + if worker.role == "sampler" and (cache_dir := vllm_cache_hostpath()): + # vLLM keeps its torch.compile artifacts and kernel autotune results under + # VLLM_CACHE_ROOT, keyed by model, GPU and engine config. In the container + # they die with the pod and every sampler start compiles again: measured + # 28-37 s on an H100 and 122 s on an L4 for gemma-4-e2b. A directory on + # the node's disk keeps them for the next sampler of that model there. + template["spec"]["containers"][0]["volumeMounts"].append({"name": "vllm-cache", "mountPath": VLLM_CACHE_MOUNT}) + template["spec"]["volumes"].append({"name": "vllm-cache", "hostPath": {"path": cache_dir, "type": "DirectoryOrCreate"}}) return template +VLLM_CACHE_MOUNT = "/var/cache/openrl/vllm" + + +def vllm_cache_hostpath() -> str | None: + """Where samplers keep vLLM's compile cache on the node, or None when turned off.""" + value = os.getenv("OPEN_RL_VLLM_CACHE_HOSTPATH", "/var/lib/openrl/vllm-cache").strip() + return None if value.lower() in ("", "off", "none") else value + + def accelerator_spec(worker: Worker) -> dict[str, Any]: """A torchrun group asks for its devices, each sized for a whole replica.""" if worker.devices > 1: diff --git a/tests/test_scheduler_worker_manager.py b/tests/test_scheduler_worker_manager.py index 40becf0e..2cc320a0 100644 --- a/tests/test_scheduler_worker_manager.py +++ b/tests/test_scheduler_worker_manager.py @@ -130,6 +130,37 @@ def test_mutable_worker_images_use_the_requested_pull_policy(self) -> None: # Reusing kind-dev must fetch the rebuilt worker rather than its cached predecessor. self.assertEqual(container["imagePullPolicy"], "Always") + def test_workers_default_to_if_not_present_so_a_prepulled_image_counts(self) -> None: + s = self.store_with("job-lora-1", {"base_model": "Qwen/Qwen2.5-0.5B", "fine_tuning_type": "lora"}) + with patch("server.worker_manager.get_state_store", return_value=s), patch.dict(os.environ, {}, clear=False): + os.environ.pop("OPEN_RL_WORKER_IMAGE_PULL_POLICY", None) + self.manager.ensure("job-lora-1", "trainer") + container = self.api.created[0]["spec"]["template"]["spec"]["containers"][0] + self.assertEqual(container["imagePullPolicy"], "IfNotPresent") + + def test_samplers_keep_vllms_compile_cache_on_the_node(self) -> None: + s = self.store_with("job-lora-1", {"base_model": "Qwen/Qwen2.5-0.5B", "fine_tuning_type": "lora"}) + with patch("server.worker_manager.get_state_store", return_value=s), patch.dict(os.environ, {}, clear=False): + os.environ.pop("OPEN_RL_VLLM_CACHE_HOSTPATH", None) + self.manager.ensure("job-lora-1", "sampler") + self.manager.ensure("job-lora-1", "trainer") + sampler, trainer = (pod["spec"]["template"]["spec"] for pod in self.api.created) + env = {e["name"]: e.get("value") for e in sampler["containers"][0]["env"]} + self.assertEqual(env["VLLM_CACHE_ROOT"], "/var/cache/openrl/vllm") + self.assertIn({"name": "vllm-cache", "mountPath": "/var/cache/openrl/vllm"}, sampler["containers"][0]["volumeMounts"]) + self.assertIn({"name": "vllm-cache", "hostPath": {"path": "/var/lib/openrl/vllm-cache", "type": "DirectoryOrCreate"}}, sampler["volumes"]) + # The trainer does not run vLLM and gets nothing. + self.assertNotIn("VLLM_CACHE_ROOT", {e["name"] for e in trainer["containers"][0]["env"]}) + self.assertNotIn("vllm-cache", {v["name"] for v in trainer["volumes"]}) + + def test_the_compile_cache_can_be_turned_off(self) -> None: + s = self.store_with("job-lora-1", {"base_model": "Qwen/Qwen2.5-0.5B", "fine_tuning_type": "lora"}) + with patch("server.worker_manager.get_state_store", return_value=s), patch.dict(os.environ, {"OPEN_RL_VLLM_CACHE_HOSTPATH": "off"}): + self.manager.ensure("job-lora-1", "sampler") + spec = self.api.created[0]["spec"]["template"]["spec"] + self.assertNotIn("vllm-cache", {v["name"] for v in spec["volumes"]}) + self.assertNotIn("VLLM_CACHE_ROOT", {e["name"] for e in spec["containers"][0]["env"]}) + def test_workers_get_the_deployment_settings(self) -> None: s = self.store_with("job-lora-1", {"base_model": "Qwen/Qwen2.5-0.5B", "fine_tuning_type": "lora"}) settings = {"VLLM_MAX_MODEL_LEN": "131072", "OPEN_RL_TRAIN_TOKEN_BUDGET": "131072", "MAX_JOBS": "4"} From aa45d406092059a862917b927f31a72658560ed9 Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 14:02:49 -0700 Subject: [PATCH 08/19] scripts: one-screen watch for a demo run --- scripts/demo_watch.py | 113 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 scripts/demo_watch.py diff --git a/scripts/demo_watch.py b/scripts/demo_watch.py new file mode 100644 index 00000000..23248be2 --- /dev/null +++ b/scripts/demo_watch.py @@ -0,0 +1,113 @@ +"""One screen for the fleet demo: the jobs, their steps, the workers, the GPUs, the storage. + + python3 scripts/demo_watch.py --run drill-1 [--interval 5] [--once] + +Refreshes until Ctrl-C. Steps come from each job's metrics.jsonl on the shared +volume (one `wc -l` through the API server pod per refresh), workers from the +Workload objects, GPU occupancy from Workloads per node, storage from `df` on +the volume. Everything shown is what kubectl would show; this only puts it on +one page. +""" + +import argparse +import json +import os +import subprocess +import sys +import time +from collections import Counter +from datetime import UTC, datetime + + +def sh(*args: str, timeout: float = 25) -> str: + try: + return subprocess.run(args, check=True, capture_output=True, text=True, timeout=timeout).stdout + except Exception: + return "" + + +def ts(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None + + +def snapshot(ns: str, run: str) -> dict: + jobs = json.loads(sh("kubectl", "-n", ns, "get", "jobs", "-l", f"openrl.io/demo-run={run}", "-o", "json") or '{"items":[]}')["items"] + pods = json.loads(sh("kubectl", "-n", ns, "get", "pods", "-l", f"openrl.io/demo-run={run}", "-o", "json") or '{"items":[]}')["items"] + workloads = json.loads(sh("kubectl", "-n", ns, "get", "workloads", "-o", "json") or '{"items":[]}')["items"] + names = [j["metadata"]["name"] for j in jobs] + script = ( + f"cd /mnt/shared/open-rl/runs/demo/{run} 2>/dev/null || exit 0; " + + "for n in " + + " ".join(names) + + "; do f=$(ls $n/open-rl-tmp/*_gsm8k_rl/metrics.jsonl 2>/dev/null | head -1); " + + 'if [ -n "$f" ]; then echo "$n $(wc -l < $f)"; fi; done; df -h /mnt/shared | tail -1' + ) + raw = sh("kubectl", "-n", ns, "exec", "deploy/open-rl-api-server", "--", "sh", "-c", script) if names else "" + steps, disk = {}, "" + for line in raw.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0] in names: + steps[parts[0]] = int(parts[1]) + elif line.startswith("10.") or "/mnt/shared" in line: + disk = line + pod_by_job = {p["metadata"]["labels"].get("job-name"): p for p in pods} + return {"jobs": jobs, "pod_by_job": pod_by_job, "workloads": workloads, "steps": steps, "disk": disk} + + +def render(snap: dict, run: str, budget: float, target_steps: int) -> str: + now = datetime.now(UTC) + lines = [f"OpenRL fleet demo run={run} {now.strftime('%H:%M:%S')} UTC", ""] + lines.append(f"{'job':44} {'status':10} {'age':>6} {'steps':>5}") + done_in_time = total = 0 + for job in sorted(snap["jobs"], key=lambda j: j["metadata"]["name"]): + name = job["metadata"]["name"] + pod = snap["pod_by_job"].get(name, {}) + start = ts(pod.get("status", {}).get("startTime")) or ts(job["metadata"]["creationTimestamp"]) + age = (now - start).total_seconds() if start else 0 + status = ( + "done" if job["status"].get("succeeded") else "FAILED" if job["status"].get("failed") else pod.get("status", {}).get("phase", "pending").lower() + ) + n = snap["steps"].get(name, 0) + total += 1 + if n >= target_steps and age <= budget or (job["status"].get("succeeded") and n >= target_steps and age <= budget): + done_in_time += 1 + bar = "#" * n + "." * max(0, target_steps - n) + lines.append(f"{name[:44]:44} {status:10} {age:6.0f} {n:>3} {bar}") + lines.append("") + by_node: Counter = Counter() + phases: Counter = Counter() + for wl in snap["workloads"]: + st = wl.get("status", {}) + phases[st.get("phase", "Pending")] += 1 + if st.get("nodeName"): + by_node[st["nodeName"].replace("gke-open-rl-dra-", "")] += st.get("deviceCount") or 1 + lines.append(f"workers: {len(snap['workloads'])} " + " ".join(f"{k}={v}" for k, v in sorted(phases.items()))) + lines.append("GPUs in use by node: " + ", ".join(f"{n}:{c}" for n, c in sorted(by_node.items()))) + lines.append(f"storage: {snap['disk'] or 'n/a'}") + lines.append("") + lines.append(f"{done_in_time}/{total} jobs at >= {target_steps} steps within {budget:.0f} s") + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run", required=True) + parser.add_argument("--namespace", default="openrl-system") + parser.add_argument("--interval", type=float, default=5.0) + parser.add_argument("--budget", type=float, default=180.0) + parser.add_argument("--steps", type=int, default=3) + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + while True: + text = render(snapshot(args.namespace, args.run), args.run, args.budget, args.steps) + if args.once: + print(text) + return + os.system("clear") + sys.stdout.write(text + "\n") + sys.stdout.flush() + time.sleep(args.interval) + + +if __name__ == "__main__": + main() From 459e718a313356ee67fc074bc5b45de49b523330 Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 14:21:31 -0700 Subject: [PATCH 09/19] deploy: let the vLLM compile cache live on a shared directory (OPEN_RL_VLLM_CACHE_DIR) --- src/server/scheduler_worker_manager.py | 14 +++++++++++--- tests/test_scheduler_worker_manager.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/server/scheduler_worker_manager.py b/src/server/scheduler_worker_manager.py index c0b59ea6..c9a11d9b 100644 --- a/src/server/scheduler_worker_manager.py +++ b/src/server/scheduler_worker_manager.py @@ -126,8 +126,14 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: # No other worker shares an exclusive worker's GPUs, so it never parks. if worker.exclusive: values["OPEN_RL_TIME_SLICING"] = "off" - if worker.role == "sampler" and vllm_cache_hostpath(): - values["VLLM_CACHE_ROOT"] = VLLM_CACHE_MOUNT + if worker.role == "sampler": + # OPEN_RL_VLLM_CACHE_DIR names a directory the worker already sees, such + # as one on the shared volume, so a cache can be fleet-wide; otherwise the + # node-local hostPath mounted below. + if shared := os.getenv("OPEN_RL_VLLM_CACHE_DIR"): + values["VLLM_CACHE_ROOT"] = shared + elif vllm_cache_hostpath(): + values["VLLM_CACHE_ROOT"] = VLLM_CACHE_MOUNT env: list[dict[str, Any]] = [{"name": name, "value": value} for name, value in values.items()] env.append({"name": "OPEN_RL_ACCEL_TIMESLICER_HOST", "valueFrom": {"fieldRef": {"fieldPath": "status.hostIP"}}}) return env @@ -183,7 +189,9 @@ def pod_template(worker: Worker) -> dict[str, Any]: # worker start would then re-pull. The prepull DaemonSet puts the image on # the node ahead of time; IfNotPresent is what makes that count. template["spec"]["containers"][0]["imagePullPolicy"] = os.getenv("OPEN_RL_WORKER_IMAGE_PULL_POLICY", "IfNotPresent") - if worker.role == "sampler" and (cache_dir := vllm_cache_hostpath()): + if worker.role == "sampler" and os.getenv("OPEN_RL_VLLM_CACHE_DIR"): + pass # a path inside an existing mount, typically the shared volume; nothing to add + elif worker.role == "sampler" and (cache_dir := vllm_cache_hostpath()): # vLLM keeps its torch.compile artifacts and kernel autotune results under # VLLM_CACHE_ROOT, keyed by model, GPU and engine config. In the container # they die with the pod and every sampler start compiles again: measured diff --git a/tests/test_scheduler_worker_manager.py b/tests/test_scheduler_worker_manager.py index 2cc320a0..1d4a4d7d 100644 --- a/tests/test_scheduler_worker_manager.py +++ b/tests/test_scheduler_worker_manager.py @@ -153,6 +153,18 @@ def test_samplers_keep_vllms_compile_cache_on_the_node(self) -> None: self.assertNotIn("VLLM_CACHE_ROOT", {e["name"] for e in trainer["containers"][0]["env"]}) self.assertNotIn("vllm-cache", {v["name"] for v in trainer["volumes"]}) + def test_the_compile_cache_can_live_on_the_shared_volume_instead(self) -> None: + s = self.store_with("job-lora-1", {"base_model": "Qwen/Qwen2.5-0.5B", "fine_tuning_type": "lora"}) + with ( + patch("server.worker_manager.get_state_store", return_value=s), + patch.dict(os.environ, {"OPEN_RL_VLLM_CACHE_DIR": "/mnt/shared/open-rl/vllm-cache"}), + ): + self.manager.ensure("job-lora-1", "sampler") + spec = self.api.created[0]["spec"]["template"]["spec"] + env = {e["name"]: e.get("value") for e in spec["containers"][0]["env"]} + self.assertEqual(env["VLLM_CACHE_ROOT"], "/mnt/shared/open-rl/vllm-cache") + self.assertNotIn("vllm-cache", {v["name"] for v in spec["volumes"]}) + def test_the_compile_cache_can_be_turned_off(self) -> None: s = self.store_with("job-lora-1", {"base_model": "Qwen/Qwen2.5-0.5B", "fine_tuning_type": "lora"}) with patch("server.worker_manager.get_state_store", return_value=s), patch.dict(os.environ, {"OPEN_RL_VLLM_CACHE_HOSTPATH": "off"}): From 0e025fa69e78ff81d3cd0466b56009812ebb4c34 Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 14:34:38 -0700 Subject: [PATCH 10/19] deploy: share vLLM's compile cache on the shared volume by default Measured on open-rl-dra: a sampler of a model already compiled on any node loads the AOT artifact in 1.1 s instead of compiling for 56-67 s, and its engine initializes in 20 s instead of 88-106 s. The cache is 61 MB per model and GPU type. The per-node hostPath remains for installs without a shared volume (unset OPEN_RL_VLLM_CACHE_DIR). --- k8s/deploy/base/api-server.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/k8s/deploy/base/api-server.yaml b/k8s/deploy/base/api-server.yaml index de9f6c17..09e435db 100644 --- a/k8s/deploy/base/api-server.yaml +++ b/k8s/deploy/base/api-server.yaml @@ -53,6 +53,11 @@ spec: key: BASE_MODEL - name: OPEN_RL_TMP_DIR value: "/mnt/shared/open-rl" + # vLLM's compile cache, shared by every sampler in the fleet: the first + # sampler of a model on a GPU type compiles (about a minute), the rest + # load the result in a second. Unset it to keep a per-node hostPath instead. + - name: OPEN_RL_VLLM_CACHE_DIR + value: "/mnt/shared/open-rl/vllm-cache" # vLLM sampling enabled on cluster. - name: SAMPLING_BACKEND value: "vllm" From 8736ff044bccba1aa8865e199afa5ba710a7f9fb Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 14:54:55 -0700 Subject: [PATCH 11/19] sampler: capture CUDA graphs for power-of-two batch sizes only vLLM's default list captured dozens of sizes on every sampler start, 12-40 s of it (runs/2026-10-17_demo_baselines). GRPO sends a few shapes that pad to the next power of two; VLLM_CUDAGRAPH_CAPTURE_SIZES overrides the list or keeps vLLM's with "default". The deployment can now also forward VLLM_ENABLE_V1_MULTIPROCESSING to try the in-process engine. --- src/server/scheduler_worker_manager.py | 9 ++++++++- src/server/vllm_options.py | 26 ++++++++++++++++++++++++++ src/server/vllm_sampler.py | 3 ++- tests/test_vllm_options.py | 19 +++++++++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/server/scheduler_worker_manager.py b/src/server/scheduler_worker_manager.py index c9a11d9b..09a17cbe 100644 --- a/src/server/scheduler_worker_manager.py +++ b/src/server/scheduler_worker_manager.py @@ -120,7 +120,14 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: } # MAX_JOBS caps FlashInfer's JIT build, which otherwise runs one ~3GB # compiler per core and blows through the pod's host memory limit. - for name in ("VLLM_GPU_MEMORY_UTILIZATION", "VLLM_MAX_MODEL_LEN", "OPEN_RL_TRAIN_TOKEN_BUDGET", "MAX_JOBS"): + for name in ( + "VLLM_GPU_MEMORY_UTILIZATION", + "VLLM_MAX_MODEL_LEN", + "VLLM_CUDAGRAPH_CAPTURE_SIZES", + "VLLM_ENABLE_V1_MULTIPROCESSING", + "OPEN_RL_TRAIN_TOKEN_BUDGET", + "MAX_JOBS", + ): if os.getenv(name): values[name] = os.environ[name] # No other worker shares an exclusive worker's GPUs, so it never parks. diff --git a/src/server/vllm_options.py b/src/server/vllm_options.py index a62a6dbb..d742239c 100644 --- a/src/server/vllm_options.py +++ b/src/server/vllm_options.py @@ -2,6 +2,7 @@ import os from collections.abc import Sequence +from typing import Any def text_only_engine_kwargs() -> dict: @@ -83,3 +84,28 @@ def detected_device_memory_bytes() -> int: except Exception: pass return 0 + + +# Batch sizes vLLM records a CUDA graph for. Its default list has dozens of +# entries up to max_num_seqs and capturing them took 12-40 s on every sampler +# start (runs/2026-10-17_demo_baselines). A GRPO step sends a handful of +# shapes; powers of two up to max_num_seqs cover them with padding and cut the +# capture to a few sizes. VLLM_CUDAGRAPH_CAPTURE_SIZES: "default" keeps vLLM's +# list, otherwise a comma-separated list of sizes. +def cudagraph_capture_sizes(max_num_seqs: int) -> list[int] | None: + raw = os.getenv("VLLM_CUDAGRAPH_CAPTURE_SIZES", "").strip().lower() + if raw == "default": + return None + if raw: + sizes = sorted({int(s) for s in raw.split(",") if s.strip()}) + else: + sizes, size = [], 1 + while size <= max_num_seqs: + sizes.append(size) + size *= 2 + return [s for s in sizes if s <= max_num_seqs] or [max_num_seqs] + + +def compilation_kwargs(max_num_seqs: int) -> dict[str, Any]: + sizes = cudagraph_capture_sizes(max_num_seqs) + return {} if sizes is None else {"compilation_config": {"cudagraph_capture_sizes": sizes}} diff --git a/src/server/vllm_sampler.py b/src/server/vllm_sampler.py index 84b461f0..811d1a65 100644 --- a/src/server/vllm_sampler.py +++ b/src/server/vllm_sampler.py @@ -27,7 +27,7 @@ from accel_timeslicer.time_slicer import TimeSlicerClient, time_slicer_client_from_env, time_slicing_enabled, workload_from_env from accel_timeslicer.workload import SAMPLER_CLAIM, WorkloadRef, local_workload_name from server.store import RequestStore, StateStore, get_state_store, get_store -from server.vllm_options import gpu_memory_utilization, sampler_batch_limits, split_stop, text_only_engine_kwargs +from server.vllm_options import compilation_kwargs, gpu_memory_utilization, sampler_batch_limits, split_stop, text_only_engine_kwargs tracer = trace.get_tracer("vllm.inference.worker") SHUTDOWN_SENTINEL = "SHUTDOWN_SENTINEL" @@ -52,6 +52,7 @@ def engine_kwargs_from_env(fft_enabled: bool) -> dict[str, Any]: "max_model_len": int(os.getenv("VLLM_MAX_MODEL_LEN", "8192")), **sampler_batch_limits(), "gpu_memory_utilization": gpu_memory_utilization(), + **compilation_kwargs(sampler_batch_limits()["max_num_seqs"]), "enable_prefix_caching": False, "enforce_eager": os.getenv("VLLM_ENFORCE_EAGER", "0") == "1", **text_only_engine_kwargs(), diff --git a/tests/test_vllm_options.py b/tests/test_vllm_options.py index d81f23dc..d43af9cc 100644 --- a/tests/test_vllm_options.py +++ b/tests/test_vllm_options.py @@ -2,6 +2,7 @@ import unittest from unittest.mock import patch +from server import vllm_options from server.vllm_options import gpu_memory_utilization, split_stop @@ -46,3 +47,21 @@ def test_placement_claim_does_not_cap_the_device(self) -> None: def test_without_a_budget_the_default_holds(self) -> None: with patch.dict(os.environ, {}, clear=True): self.assertEqual(gpu_memory_utilization(), 0.90) + + +class CudagraphCaptureSizesTest(unittest.TestCase): + def test_powers_of_two_up_to_the_sequence_limit_by_default(self): + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("VLLM_CUDAGRAPH_CAPTURE_SIZES", None) + self.assertEqual(vllm_options.cudagraph_capture_sizes(64), [1, 2, 4, 8, 16, 32, 64]) + self.assertEqual(vllm_options.cudagraph_capture_sizes(256)[-1], 256) + self.assertEqual(vllm_options.compilation_kwargs(8), {"compilation_config": {"cudagraph_capture_sizes": [1, 2, 4, 8]}}) + + def test_an_explicit_list_is_sorted_and_capped(self): + with patch.dict(os.environ, {"VLLM_CUDAGRAPH_CAPTURE_SIZES": "64, 8,16,512"}): + self.assertEqual(vllm_options.cudagraph_capture_sizes(128), [8, 16, 64]) + + def test_default_keeps_vllms_own_list(self): + with patch.dict(os.environ, {"VLLM_CUDAGRAPH_CAPTURE_SIZES": "default"}): + self.assertIsNone(vllm_options.cudagraph_capture_sizes(64)) + self.assertEqual(vllm_options.compilation_kwargs(64), {}) From d5361edcf913544ada9231efd7dd0c8e954b8e8e Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 18:05:35 -0700 Subject: [PATCH 12/19] sampler: keep room for the weight update's dense staging copy An FFT sampler applies each sparse delta through vLLM's checkpoint patch loader, which stages a dense copy of every patched checkpoint tensor on the device. gemma-4-e2b's per-layer input embedding is 4.38 GiB of that, and vLLM's default share leaves 2.2 GiB free on an L4: the sampler died at its first update (runs/2026-10-17_demo_baselines, solo gemma probe). The sampler now reads the largest tensor from the checkpoint's safetensors headers and takes that much, plus a margin, off vLLM's share of the device, never below a floor; OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB overrides it. The estimator adds the same room to the FFT sampler's placement claim from a per-model table, so the claim and the runtime agree. LoRA samplers never update base weights and are unchanged. --- src/server/estimator.py | 40 +++++++++++++- src/server/vllm_options.py | 107 +++++++++++++++++++++++++++++++++++-- src/server/vllm_sampler.py | 13 ++++- tests/test_estimator.py | 27 +++++++++- tests/test_vllm_options.py | 64 ++++++++++++++++++++++ 5 files changed, 243 insertions(+), 8 deletions(-) diff --git a/src/server/estimator.py b/src/server/estimator.py index 37383b3c..77bb37a3 100644 --- a/src/server/estimator.py +++ b/src/server/estimator.py @@ -41,6 +41,27 @@ "gemma-4-e2b": 7_168, # 7 of 35 layers are full attention, 1 kv head, head dim 256 "gemma-4-e4b": 14_336, # 7 of 42 layers are full attention, 2 kv heads, head dim 256 } +# Elements in the largest checkpoint tensor. A full-fine-tuning sampler applies +# each weight update through vLLM's patch loader, which stages a dense copy of +# every patched checkpoint tensor on the device, so the sampler must keep that +# much free on top of what vLLM profiles. Read from the safetensors headers for +# the models in the cache; the rest are vocab x hidden from config.json. Gemma-4 +# models carry a per-layer input embedding, vocab x (256 x layers), that is far +# larger than the token embedding. +MODEL_TO_LARGEST_TENSOR_ELEMENTS: dict[str, int] = { + "qwen2.5-0.5b": 136_134_656, # embed_tokens [151936, 896] + "qwen3-0.6b": 155_582_464, # embed_tokens [151936, 1024] + "qwen2.5-1.5b": 233_373_696, # [151936, 1536] + "qwen3-1.7b": 311_164_928, # [151936, 2048] + "qwen3-4b": 388_956_160, # embed_tokens [151936, 2560] + "qwen2.5-7b": 544_997_376, # [152064, 3584] + "qwen3-8b": 622_329_856, # embed_tokens [151936, 4096] + "qwen3.5-9b": 1_017_118_720, # [248320, 4096] + "qwen3.5-27b": 1_271_398_400, # [248320, 5120] + "gemma-3-1b": 301_989_888, # [262144, 1152] + "gemma-4-e2b": 2_348_810_240, # embed_tokens_per_layer [262144, 8960] + "gemma-4-e4b": 2_818_572_288, # embed_tokens_per_layer [262144, 10752] +} UNKNOWN_MODEL = "qwen3-8b" # unknown models are sized large, not small VARIANT_SUFFIXES = ("-instruct", "-it", "-pt", "-base", "-chat") @@ -78,6 +99,12 @@ def parameter_count(base_model: str) -> int | None: SAMPLER_LORA_SLOT_BYTES = GIB SAMPLER_LORA_SLOT_BYTES_PER_PARAM = 0.25 SAMPLER_KV_TOKENS = 8 * 8192 # eight max-length requests in flight +# A full-fine-tuning sampler's weight update: the largest checkpoint tensor +# staged densely in bf16, plus the sparse delta and the loader's working set. +# Measured: gemma-4-e2b's update asked for 4.38 GiB on an L4 that had 1.86 GiB +# free and died (runs/2026-10-17_demo_baselines). +SAMPLER_WEIGHT_UPDATE_BYTES_PER_ELEMENT = 2 +SAMPLER_WEIGHT_UPDATE_MARGIN_BYTES = GIB // 2 # Parked in host memory: fft trainer 12 B/param + a weight copy in flight; # plus process overhead. Measured: 0.5B trainer 28Gi, sampler 20Gi; 8B FFT # sampler 39Gi steady; 7B FFT trainer OOM-killed at 110Gi. @@ -108,14 +135,23 @@ def resources(self) -> dict: return {"requests": {"memory": gib(self.host_request_bytes)}, "limits": {"memory": gib(self.host_limit_bytes)}} -def sampler_device_bytes(params: int, kv_bytes_per_token: int, kind: str) -> int: +def sampler_device_bytes(params: int, kv_bytes_per_token: int, kind: str, largest_tensor_elements: int = 0) -> int: device = params * SAMPLER_WEIGHT_BYTES_PER_PARAM device += SAMPLER_OVERHEAD_BYTES + int(params * SAMPLER_OVERHEAD_BYTES_PER_PARAM) if kind == "lora": device += SAMPLER_LORA_SLOT_BYTES + int(params * SAMPLER_LORA_SLOT_BYTES_PER_PARAM) + else: + device += weight_update_bytes(largest_tensor_elements) return device + kv_bytes_per_token * SAMPLER_KV_TOKENS +def weight_update_bytes(largest_tensor_elements: int) -> int: + """What a full-fine-tuning sampler keeps free for applying a weight update.""" + if largest_tensor_elements <= 0: + return 0 + return largest_tensor_elements * SAMPLER_WEIGHT_UPDATE_BYTES_PER_ELEMENT + SAMPLER_WEIGHT_UPDATE_MARGIN_BYTES + + def footprint(base_model: str, fine_tuning_type: str, role: str) -> Footprint: model = normalize_model_id(base_model) if model not in MODEL_TO_PARAM_COUNT: @@ -126,6 +162,6 @@ def footprint(base_model: str, fine_tuning_type: str, role: str) -> Footprint: if role == "trainer": device = params * TRAINER_DEVICE_BYTES_PER_PARAM[kind] + TRAINER_DEVICE_RESERVE_BYTES else: - device = sampler_device_bytes(params, MODEL_TO_KV_BYTES_PER_TOKEN[model], kind) + device = sampler_device_bytes(params, MODEL_TO_KV_BYTES_PER_TOKEN[model], kind, MODEL_TO_LARGEST_TENSOR_ELEMENTS[model]) host = params * HOST_BYTES_PER_PARAM[(kind, role)] + HOST_OVERHEAD_BYTES[role] return Footprint(device, host, int(host * HOST_LIMIT_FACTOR)) diff --git a/src/server/vllm_options.py b/src/server/vllm_options.py index d742239c..13e37604 100644 --- a/src/server/vllm_options.py +++ b/src/server/vllm_options.py @@ -1,6 +1,8 @@ """Engine and sampling options shared by the two vLLM sampler workers.""" +import json import os +import struct from collections.abc import Sequence from typing import Any @@ -37,8 +39,32 @@ def split_stop(stop: str | Sequence[str] | Sequence[int] | None) -> tuple[list[s return (strings or None), (token_ids or None) -def gpu_memory_utilization() -> float: - """vLLM's share of the device: an explicit VLLM_GPU_MEMORY_UTILIZATION, else 0.90. +DEFAULT_GPU_MEMORY_UTILIZATION = 0.90 +# vLLM must keep at least this much of the device, whatever the headroom asks. +MIN_GPU_MEMORY_UTILIZATION = 0.40 +# What a weight update needs beyond the largest staged tensor: the sparse +# delta's values and indices, and the loader's working set. +WEIGHT_UPDATE_MARGIN_BYTES = 512 * 2**20 +SAFETENSORS_DTYPE_BYTES = { + "F64": 8, + "I64": 8, + "U64": 8, + "F32": 4, + "I32": 4, + "U32": 4, + "BF16": 2, + "F16": 2, + "I16": 2, + "U16": 2, + "I8": 1, + "U8": 1, + "BOOL": 1, +} + + +def gpu_memory_utilization(headroom_bytes: int = 0, device_memory_bytes: int | None = None) -> float: + """vLLM's share of the device: an explicit VLLM_GPU_MEMORY_UTILIZATION, else 0.90 + less whatever headroom the caller must keep free. OPEN_RL_ACCELERATOR_MEMORY is deliberately not consulted. It is the placement claim the scheduler sized this worker for, not a runtime cap: workers that @@ -46,11 +72,86 @@ def gpu_memory_utilization() -> float: use all of it. Sizing vLLM from the claim starved the sampler of KV cache on large devices and made Gemma-4's first engine init fail when the cold torch.compile transient did not fit in the claim. + + Headroom is different: it is memory a later transient needs that vLLM's own + profiling never sees. An FFT sampler applies each weight update through + vLLM's checkpoint patch loader, which stages a dense copy of every patched + checkpoint tensor on the device; gemma-4-e2b's per-layer embedding is 4.4 GiB + of that, and on an L4 the 10% vLLM leaves free is 2.2 GiB. The sampler died at + its first update (runs/2026-10-17_demo_baselines). The headroom comes off + vLLM's share so the update fits, and never pushes the share below the floor. """ explicit = os.getenv("VLLM_GPU_MEMORY_UTILIZATION") if explicit: return float(explicit) - return 0.90 + if headroom_bytes <= 0: + return DEFAULT_GPU_MEMORY_UTILIZATION + if device_memory_bytes is None: + device_memory_bytes = detected_device_memory_bytes() + if device_memory_bytes <= 0: + return DEFAULT_GPU_MEMORY_UTILIZATION + share = DEFAULT_GPU_MEMORY_UTILIZATION - headroom_bytes / device_memory_bytes + return round(max(MIN_GPU_MEMORY_UTILIZATION, min(DEFAULT_GPU_MEMORY_UTILIZATION, share)), 4) + + +def weight_update_headroom_bytes(model: str) -> int: + """Device memory a full-fine-tuning sampler must keep free for weight updates. + + OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB overrides the estimate; "0" disables it. + Otherwise it is the largest tensor in the model's checkpoint, the dense + staging copy the patch loader allocates, plus a fixed margin. The checkpoint + is already in the local Hugging Face cache because the engine loads from it. + """ + explicit = os.getenv("OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB") + if explicit: + return int(float(explicit) * 2**30) + largest = largest_checkpoint_tensor_bytes(model) + if largest <= 0: + return 0 + return largest + WEIGHT_UPDATE_MARGIN_BYTES + + +def largest_checkpoint_tensor_bytes(model: str) -> int: + """The biggest tensor in the model's safetensors files, from their headers only. + + Returns 0 when the checkpoint cannot be found locally or holds no safetensors. + """ + directory = model if os.path.isdir(model) else local_snapshot(model) + if not directory: + return 0 + largest = 0 + for name in sorted(os.listdir(directory)): + if not name.endswith(".safetensors"): + continue + try: + largest = max(largest, largest_tensor_in_header(os.path.join(directory, name))) + except (OSError, ValueError, KeyError): + continue + return largest + + +def local_snapshot(model: str) -> str | None: + try: + from huggingface_hub import snapshot_download + + return snapshot_download(model, local_files_only=True, allow_patterns=["*.safetensors"]) + except Exception: + return None + + +def largest_tensor_in_header(path: str) -> int: + with open(path, "rb") as f: + (header_len,) = struct.unpack(" dict[str, int]: diff --git a/src/server/vllm_sampler.py b/src/server/vllm_sampler.py index 811d1a65..40ea91bb 100644 --- a/src/server/vllm_sampler.py +++ b/src/server/vllm_sampler.py @@ -27,7 +27,14 @@ from accel_timeslicer.time_slicer import TimeSlicerClient, time_slicer_client_from_env, time_slicing_enabled, workload_from_env from accel_timeslicer.workload import SAMPLER_CLAIM, WorkloadRef, local_workload_name from server.store import RequestStore, StateStore, get_state_store, get_store -from server.vllm_options import compilation_kwargs, gpu_memory_utilization, sampler_batch_limits, split_stop, text_only_engine_kwargs +from server.vllm_options import ( + compilation_kwargs, + gpu_memory_utilization, + sampler_batch_limits, + split_stop, + text_only_engine_kwargs, + weight_update_headroom_bytes, +) tracer = trace.get_tracer("vllm.inference.worker") SHUTDOWN_SENTINEL = "SHUTDOWN_SENTINEL" @@ -51,7 +58,9 @@ def engine_kwargs_from_env(fft_enabled: bool) -> dict[str, Any]: "enable_lora": not fft_enabled, "max_model_len": int(os.getenv("VLLM_MAX_MODEL_LEN", "8192")), **sampler_batch_limits(), - "gpu_memory_utilization": gpu_memory_utilization(), + # An FFT sampler keeps room for the dense staging copy each weight update + # allocates; a LoRA sampler never updates base weights. + "gpu_memory_utilization": gpu_memory_utilization(weight_update_headroom_bytes(model_name) if fft_enabled else 0), **compilation_kwargs(sampler_batch_limits()["max_num_seqs"]), "enable_prefix_caching": False, "enforce_eager": os.getenv("VLLM_ENFORCE_EAGER", "0") == "1", diff --git a/tests/test_estimator.py b/tests/test_estimator.py index ebd14d1d..f8e79b43 100644 --- a/tests/test_estimator.py +++ b/tests/test_estimator.py @@ -2,7 +2,16 @@ import unittest from unittest.mock import patch -from server.estimator import GIB, MODEL_TO_KV_BYTES_PER_TOKEN, SAMPLER_KV_TOKENS, footprint, normalize_model_id, parameter_count +from server.estimator import ( + GIB, + MODEL_TO_KV_BYTES_PER_TOKEN, + MODEL_TO_LARGEST_TENSOR_ELEMENTS, + SAMPLER_KV_TOKENS, + footprint, + normalize_model_id, + parameter_count, + weight_update_bytes, +) def kv_cache_bytes(model: str) -> int: @@ -35,6 +44,22 @@ def test_sampler_matches_the_live_samplers(self) -> None: predicted_kv = kv_cache_bytes(model) - (fp.accelerator_bytes - budget * GIB) self.assertAlmostEqual(predicted_kv / GIB, kv, delta=0.3, msg=model) + def test_fft_sampler_keeps_room_for_its_weight_update(self) -> None: + # gemma-4-e2b's sparse update stages its 4.38 GiB per-layer embedding + # densely on the device; sized without it the sampler landed on an L4 and + # died at the first update. The claim carries it, and still fits the L4 + # (so does vLLM's share once it is told to leave the room, see + # vllm_options.gpu_memory_utilization). LoRA samplers never update base + # weights and pay nothing. + room = weight_update_bytes(MODEL_TO_LARGEST_TENSOR_ELEMENTS["gemma-4-e2b"]) + self.assertGreater(room, 4 * GIB) + full = footprint("google/gemma-4-e2b", "full", "sampler").accelerator_bytes + self.assertGreater(full, 16 * GIB) + self.assertLess(full, 22 * GIB) + self.assertLess( + footprint("Qwen/Qwen3-0.6B", "full", "sampler").accelerator_bytes, footprint("Qwen/Qwen3-0.6B", "lora", "sampler").accelerator_bytes + ) + def test_sampler_kv_cache_grows_with_the_model(self) -> None: # An 8B LoRA sampler sized at 22Gi landed on an L4 with 0.23 GiB of KV # cache left and crash-looped; it belongs on the 80GB tier. 4B still fits, diff --git a/tests/test_vllm_options.py b/tests/test_vllm_options.py index d43af9cc..0a26f7dc 100644 --- a/tests/test_vllm_options.py +++ b/tests/test_vllm_options.py @@ -1,10 +1,15 @@ +import json import os +import struct +import tempfile import unittest from unittest.mock import patch from server import vllm_options from server.vllm_options import gpu_memory_utilization, split_stop +GIB = 1024**3 + class SplitStopTest(unittest.TestCase): def test_none(self): @@ -48,6 +53,65 @@ def test_without_a_budget_the_default_holds(self) -> None: with patch.dict(os.environ, {}, clear=True): self.assertEqual(gpu_memory_utilization(), 0.90) + def test_headroom_comes_off_vllms_share(self) -> None: + # gemma-4-e2b's weight update stages a 4.38 GiB tensor; on a 22 GiB L4 vLLM + # must leave that free on top of its own 10%, and on an 80 GB H100 a little. + with patch.dict(os.environ, {}, clear=True): + l4 = gpu_memory_utilization(headroom_bytes=int(4.88 * GIB), device_memory_bytes=int(22.03 * GIB)) + h100 = gpu_memory_utilization(headroom_bytes=int(4.88 * GIB), device_memory_bytes=80 * GIB) + self.assertAlmostEqual(l4, 0.90 - 4.88 / 22.03, places=3) + self.assertAlmostEqual(h100, 0.90 - 4.88 / 80, places=3) + self.assertLess((0.90 - l4) * 22.03, 4.88 + 0.01) + + def test_headroom_never_pushes_the_share_below_the_floor(self) -> None: + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(gpu_memory_utilization(headroom_bytes=20 * GIB, device_memory_bytes=22 * GIB), vllm_options.MIN_GPU_MEMORY_UTILIZATION) + + def test_explicit_fraction_ignores_headroom(self) -> None: + with patch.dict(os.environ, {"VLLM_GPU_MEMORY_UTILIZATION": "0.5"}, clear=True): + self.assertEqual(gpu_memory_utilization(headroom_bytes=10 * GIB, device_memory_bytes=22 * GIB), 0.5) + + def test_unknown_device_keeps_the_default(self) -> None: + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(gpu_memory_utilization(headroom_bytes=5 * GIB, device_memory_bytes=0), 0.90) + + +def write_safetensors(path: str, tensors: dict[str, tuple[str, list[int]]]) -> None: + """A safetensors file with the given header and zero-filled data.""" + header, offset = {}, 0 + for name, (dtype, shape) in tensors.items(): + numel = 1 + for dim in shape: + numel *= dim + size = numel * vllm_options.SAFETENSORS_DTYPE_BYTES[dtype] + header[name] = {"dtype": dtype, "shape": shape, "data_offsets": [offset, offset + size]} + offset += size + raw = json.dumps(header).encode() + with open(path, "wb") as f: + f.write(struct.pack(" None: + with tempfile.TemporaryDirectory() as d: + write_safetensors(os.path.join(d, "model-00001.safetensors"), {"embed": ("BF16", [1024, 64]), "norm": ("F32", [64])}) + write_safetensors(os.path.join(d, "model-00002.safetensors"), {"head": ("BF16", [2048, 64])}) + self.assertEqual(vllm_options.largest_checkpoint_tensor_bytes(d), 2048 * 64 * 2) + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(vllm_options.weight_update_headroom_bytes(d), 2048 * 64 * 2 + vllm_options.WEIGHT_UPDATE_MARGIN_BYTES) + + def test_missing_checkpoint_means_no_headroom(self) -> None: + with patch.dict(os.environ, {}, clear=True), patch.object(vllm_options, "local_snapshot", return_value=None): + self.assertEqual(vllm_options.weight_update_headroom_bytes("nobody/no-such-model"), 0) + + def test_explicit_headroom_wins(self) -> None: + with patch.dict(os.environ, {"OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB": "1.5"}, clear=True): + self.assertEqual(vllm_options.weight_update_headroom_bytes("nobody/no-such-model"), int(1.5 * GIB)) + with patch.dict(os.environ, {"OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB": "0"}, clear=True): + self.assertEqual(vllm_options.weight_update_headroom_bytes("nobody/no-such-model"), 0) + class CudagraphCaptureSizesTest(unittest.TestCase): def test_powers_of_two_up_to_the_sequence_limit_by_default(self): From ed0f6298b53d456bc2c2ceee0dc949074e976e6b Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 18:05:35 -0700 Subject: [PATCH 13/19] scripts: give each demo client its own Hugging Face cache Twelve clients taking filelock locks in one cache directory on NFS lost a client to a stale file handle (drill 2). An init container now seeds an emptyDir per client with the dataset and the model's tokenizer and config files, skipping weight files; 38 MB for gemma-4-e2b. Also clears two lint findings in demo_report.py. --- scripts/demo_jobs.py | 34 +++++++++++++++++++++++++++++++++- scripts/demo_report.py | 13 +++++++------ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/scripts/demo_jobs.py b/scripts/demo_jobs.py index 818f7175..4cf12c74 100644 --- a/scripts/demo_jobs.py +++ b/scripts/demo_jobs.py @@ -61,7 +61,7 @@ - name: PYTHONUNBUFFERED value: "1" - name: HF_HOME - value: "/mnt/shared/open-rl/huggingface" + value: "/hf" resources: requests: cpu: "1" @@ -71,10 +71,41 @@ volumeMounts: - name: shared-storage mountPath: /mnt/shared + - name: hf-cache + mountPath: /hf + # Each client gets its own Hugging Face cache on an emptyDir, seeded from the + # shared one. Twelve clients locking one cache directory on NFS at the same + # time lost a client to "Stale file handle" in filelock (drill 2). The seed + # is the dataset plus the model's tokenizer and config files; weight files + # (anything over 64 MiB) are skipped, the client never reads them. + initContainers: + - name: seed-hf-cache + image: busybox:1.36 + command: ["sh", "-c"] + args: + - | + set -e + S=/mnt/shared/open-rl/huggingface; D=/hf; M={hub_dir} + mkdir -p $D/hub $D/datasets + [ -d $S/datasets ] && cp -a $S/datasets/. $D/datasets/ + [ -d $S/hub/datasets--openai--gsm8k ] && cp -a $S/hub/datasets--openai--gsm8k $D/hub/ + if [ -d $S/hub/$M ]; then + cd $S/hub + find $M -type d -exec mkdir -p $D/hub/{{}} \\; + find $M \\( -type l -o \\( -type f -size -65536k \\) \\) -exec cp -a {{}} $D/hub/{{}} \\; + fi + du -sh $D + volumeMounts: + - name: shared-storage + mountPath: /mnt/shared + - name: hf-cache + mountPath: /hf volumes: - name: shared-storage persistentVolumeClaim: claimName: open-rl-shared-pvc + - name: hf-cache + emptyDir: {{}} """ @@ -110,6 +141,7 @@ def main() -> None: image=args.image, scenario=f"{kind}-gsm8k-rl", model=model, + hub_dir="models--" + model.replace("/", "--"), steps=args.steps, group_size=args.group_size, groups_per_batch=args.groups_per_batch, diff --git a/scripts/demo_report.py b/scripts/demo_report.py index 4f4c9a05..cd469577 100644 --- a/scripts/demo_report.py +++ b/scripts/demo_report.py @@ -37,12 +37,10 @@ def main() -> None: pod_by_job = {p["metadata"]["labels"].get("job-name"): p for p in pods} # One cat per job through the API server pod, which mounts the volume. names = [j["metadata"]["name"] for j in jobs] + job_list = " ".join(names) script = ( - 'for n in %s; do for f in /mnt/shared/open-rl/runs/demo/%s/$n/open-rl-tmp/*_gsm8k_rl/metrics.jsonl; do [ -f "$f" ] && { echo "== $n"; cat "$f"; }; done; done' - % ( - " ".join(names), - args.run, - ) + f"for n in {job_list}; do for f in /mnt/shared/open-rl/runs/demo/{args.run}/$n/open-rl-tmp/*_gsm8k_rl/metrics.jsonl; " + 'do [ -f "$f" ] && { echo "== $n"; cat "$f"; }; done; done' ) raw = sh("kubectl", "-n", ns, "exec", "deploy/open-rl-api-server", "--", "sh", "-c", script) if names else "" metrics: dict[str, list[dict]] = {} @@ -80,7 +78,10 @@ def main() -> None: step_n = elapsed if step_n is not None and step_n <= args.budget: finished += 1 - mean = lambda key: f"{sum(float(r.get(key, 0)) for r in rows) / len(rows):.1f}" if rows else "" + + def mean(key: str, rows=rows) -> str: + return f"{sum(float(r.get(key, 0)) for r in rows) / len(rows):.1f}" if rows else "" + print( f"| {name} | {start.strftime('%H:%M:%S') if start else ''} | {status} | {len(rows)} | {'' if first is None else f'{first:.0f}'} | " f"{'' if step_n is None else f'{step_n:.0f}'} | {mean('time/sampling')} | {mean('time/train_step')} | {mean('time/total')} |" From 87417c27eb05923994faf329ec0e25e990c28aed Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 18:30:17 -0700 Subject: [PATCH 14/19] sampler: wake the KV cache after the weight update, and size the update's room at its peak vLLM's patch loader stages each patched checkpoint tensor densely, then its NaN-masked copy_ builds a bool mask and the merged result: 2.5x the tensor, all live at once. For gemma-4-e2b's per-layer embedding that is 11 GiB. Measured on an L4 (runs/2026-10-17_demo_baselines): 1x headroom still died at the 2.19 GiB mask; the update went through with 10.7 GiB free. The sampler now wakes only the weights when it takes the GPU and wakes the KV cache after ensure_weights, vLLM's documented order for RL updates, so the parked KV cache is not in the way. The runtime headroom is 2.5x the largest tensor plus a margin, and vLLM's share never drops below what the resident weights need. The estimator's claim uses the same factor: a gemma FFT sampler no longer fits a 24 GB device and goes to the 80 GB tier by its own size. --- src/server/estimator.py | 15 +++++---- src/server/vllm_options.py | 65 +++++++++++++++++++++++-------------- src/server/vllm_sampler.py | 55 +++++++++++++++++++++---------- tests/test_estimator.py | 24 +++++++------- tests/test_sampler_patch.py | 23 ++++++++++++- tests/test_vllm_options.py | 13 +++++++- 6 files changed, 132 insertions(+), 63 deletions(-) diff --git a/src/server/estimator.py b/src/server/estimator.py index 77bb37a3..11e03627 100644 --- a/src/server/estimator.py +++ b/src/server/estimator.py @@ -99,12 +99,15 @@ def parameter_count(base_model: str) -> int | None: SAMPLER_LORA_SLOT_BYTES = GIB SAMPLER_LORA_SLOT_BYTES_PER_PARAM = 0.25 SAMPLER_KV_TOKENS = 8 * 8192 # eight max-length requests in flight -# A full-fine-tuning sampler's weight update: the largest checkpoint tensor -# staged densely in bf16, plus the sparse delta and the loader's working set. -# Measured: gemma-4-e2b's update asked for 4.38 GiB on an L4 that had 1.86 GiB -# free and died (runs/2026-10-17_demo_baselines). -SAMPLER_WEIGHT_UPDATE_BYTES_PER_ELEMENT = 2 -SAMPLER_WEIGHT_UPDATE_MARGIN_BYTES = GIB // 2 +# A full-fine-tuning sampler's weight update: vLLM's patch loader stages the +# largest checkpoint tensor densely in bf16 (2 B/element), and its NaN-masked +# copy adds a bool mask (1 B) and the merged result (2 B), all live at once; +# plus the sparse delta and the loader's working set. Measured on gemma-4-e2b's +# 2.35 B-element per-layer embedding: the update died on an L4 with 1.9 and +# then 5.7 GiB free, and went through with 10.7 GiB free +# (runs/2026-10-17_demo_baselines). +SAMPLER_WEIGHT_UPDATE_BYTES_PER_ELEMENT = 5 +SAMPLER_WEIGHT_UPDATE_MARGIN_BYTES = GIB # Parked in host memory: fft trainer 12 B/param + a weight copy in flight; # plus process overhead. Measured: 0.5B trainer 28Gi, sampler 20Gi; 8B FFT # sampler 39Gi steady; 7B FFT trainer OOM-killed at 110Gi. diff --git a/src/server/vllm_options.py b/src/server/vllm_options.py index 13e37604..b8600175 100644 --- a/src/server/vllm_options.py +++ b/src/server/vllm_options.py @@ -42,9 +42,16 @@ def split_stop(stop: str | Sequence[str] | Sequence[int] | None) -> tuple[list[s DEFAULT_GPU_MEMORY_UTILIZATION = 0.90 # vLLM must keep at least this much of the device, whatever the headroom asks. MIN_GPU_MEMORY_UTILIZATION = 0.40 -# What a weight update needs beyond the largest staged tensor: the sparse -# delta's values and indices, and the loader's working set. -WEIGHT_UPDATE_MARGIN_BYTES = 512 * 2**20 +# vLLM's patch loader stages each patched checkpoint tensor densely (its bytes +# once), then its NaN-masked copy_ builds a bool mask (half) and the merged +# result (once more): 2.5x the tensor, all live at the same moment. Measured on +# gemma-4-e2b's 4.38 GiB per-layer embedding: 1x headroom still died at the +# 2.19 GiB mask, and the update went through with 10.7 GiB free. +WEIGHT_UPDATE_STAGING_FACTOR = 2.5 +# The sparse delta's values and indices and the loader's working set. +WEIGHT_UPDATE_MARGIN_BYTES = 2**30 +# vLLM's share must still hold the weights with room for a small KV cache. +WEIGHTS_RESIDENT_MARGIN_BYTES = 2**30 SAFETENSORS_DTYPE_BYTES = { "F64": 8, "I64": 8, @@ -62,7 +69,7 @@ def split_stop(stop: str | Sequence[str] | Sequence[int] | None) -> tuple[list[s } -def gpu_memory_utilization(headroom_bytes: int = 0, device_memory_bytes: int | None = None) -> float: +def gpu_memory_utilization(headroom_bytes: int = 0, device_memory_bytes: int | None = None, resident_bytes: int = 0) -> float: """vLLM's share of the device: an explicit VLLM_GPU_MEMORY_UTILIZATION, else 0.90 less whatever headroom the caller must keep free. @@ -75,11 +82,12 @@ def gpu_memory_utilization(headroom_bytes: int = 0, device_memory_bytes: int | N Headroom is different: it is memory a later transient needs that vLLM's own profiling never sees. An FFT sampler applies each weight update through - vLLM's checkpoint patch loader, which stages a dense copy of every patched - checkpoint tensor on the device; gemma-4-e2b's per-layer embedding is 4.4 GiB - of that, and on an L4 the 10% vLLM leaves free is 2.2 GiB. The sampler died at - its first update (runs/2026-10-17_demo_baselines). The headroom comes off - vLLM's share so the update fits, and never pushes the share below the floor. + vLLM's checkpoint patch loader, which stages dense copies of every patched + checkpoint tensor on the device; gemma-4-e2b's per-layer embedding needs 11 + GiB of that, and on an L4 the 10% vLLM leaves free is 2.2 GiB. The sampler + died at its first update (runs/2026-10-17_demo_baselines). The headroom comes + off vLLM's share so the update fits, and never pushes the share below the + floor or below what the resident weights need. """ explicit = os.getenv("VLLM_GPU_MEMORY_UTILIZATION") if explicit: @@ -91,43 +99,49 @@ def gpu_memory_utilization(headroom_bytes: int = 0, device_memory_bytes: int | N if device_memory_bytes <= 0: return DEFAULT_GPU_MEMORY_UTILIZATION share = DEFAULT_GPU_MEMORY_UTILIZATION - headroom_bytes / device_memory_bytes - return round(max(MIN_GPU_MEMORY_UTILIZATION, min(DEFAULT_GPU_MEMORY_UTILIZATION, share)), 4) + floor = max(MIN_GPU_MEMORY_UTILIZATION, (resident_bytes + WEIGHTS_RESIDENT_MARGIN_BYTES) / device_memory_bytes if resident_bytes > 0 else 0) + return round(max(floor, min(DEFAULT_GPU_MEMORY_UTILIZATION, share)), 4) def weight_update_headroom_bytes(model: str) -> int: """Device memory a full-fine-tuning sampler must keep free for weight updates. OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB overrides the estimate; "0" disables it. - Otherwise it is the largest tensor in the model's checkpoint, the dense - staging copy the patch loader allocates, plus a fixed margin. The checkpoint - is already in the local Hugging Face cache because the engine loads from it. + Otherwise it is the staging the patch loader allocates for the largest tensor + in the model's checkpoint, plus a fixed margin. The checkpoint is already in + the local Hugging Face cache because the engine loads from it. """ explicit = os.getenv("OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB") if explicit: return int(float(explicit) * 2**30) - largest = largest_checkpoint_tensor_bytes(model) + largest, _ = checkpoint_tensor_bytes(model) if largest <= 0: return 0 - return largest + WEIGHT_UPDATE_MARGIN_BYTES + return int(largest * WEIGHT_UPDATE_STAGING_FACTOR) + WEIGHT_UPDATE_MARGIN_BYTES def largest_checkpoint_tensor_bytes(model: str) -> int: - """The biggest tensor in the model's safetensors files, from their headers only. + return checkpoint_tensor_bytes(model)[0] - Returns 0 when the checkpoint cannot be found locally or holds no safetensors. + +def checkpoint_tensor_bytes(model: str) -> tuple[int, int]: + """(largest tensor, all tensors) in the model's safetensors files, from their headers only. + + Both are 0 when the checkpoint cannot be found locally or holds no safetensors. """ directory = model if os.path.isdir(model) else local_snapshot(model) if not directory: - return 0 - largest = 0 + return 0, 0 + largest = total = 0 for name in sorted(os.listdir(directory)): if not name.endswith(".safetensors"): continue try: - largest = max(largest, largest_tensor_in_header(os.path.join(directory, name))) + file_largest, file_total = tensor_bytes_in_header(os.path.join(directory, name)) except (OSError, ValueError, KeyError): continue - return largest + largest, total = max(largest, file_largest), total + file_total + return largest, total def local_snapshot(model: str) -> str | None: @@ -139,19 +153,20 @@ def local_snapshot(model: str) -> str | None: return None -def largest_tensor_in_header(path: str) -> int: +def tensor_bytes_in_header(path: str) -> tuple[int, int]: with open(path, "rb") as f: (header_len,) = struct.unpack(" dict[str, int]: diff --git a/src/server/vllm_sampler.py b/src/server/vllm_sampler.py index 40ea91bb..f1e7cffa 100644 --- a/src/server/vllm_sampler.py +++ b/src/server/vllm_sampler.py @@ -28,6 +28,7 @@ from accel_timeslicer.workload import SAMPLER_CLAIM, WorkloadRef, local_workload_name from server.store import RequestStore, StateStore, get_state_store, get_store from server.vllm_options import ( + checkpoint_tensor_bytes, compilation_kwargs, gpu_memory_utilization, sampler_batch_limits, @@ -60,7 +61,10 @@ def engine_kwargs_from_env(fft_enabled: bool) -> dict[str, Any]: **sampler_batch_limits(), # An FFT sampler keeps room for the dense staging copy each weight update # allocates; a LoRA sampler never updates base weights. - "gpu_memory_utilization": gpu_memory_utilization(weight_update_headroom_bytes(model_name) if fft_enabled else 0), + "gpu_memory_utilization": gpu_memory_utilization( + headroom_bytes=weight_update_headroom_bytes(model_name) if fft_enabled else 0, + resident_bytes=checkpoint_tensor_bytes(model_name)[1] if fft_enabled else 0, + ), **compilation_kwargs(sampler_batch_limits()["max_num_seqs"]), "enable_prefix_caching": False, "enforce_eager": os.getenv("VLLM_ENFORCE_EAGER", "0") == "1", @@ -128,31 +132,48 @@ def __init__(self, engine: AsyncLLMEngine) -> None: self.engine = engine self.weights_path: str | None = None self.update_failed = False + # A fresh engine is fully awake; sleep() parks weights and KV cache both. + self.kv_cache_awake = True async def wake(self) -> None: - await self.engine.wake_up(tags=["weights", "kv_cache"]) + """Bring the weights back; the KV cache follows once the weights are current. + + A weight update stages dense copies of the patched tensors on the device + (gemma-4-e2b: 11 GiB for its per-layer embedding). With the KV cache still + parked that memory is free; woken first, it is not, and the update died on + an L4 (runs/2026-10-17_demo_baselines). This is vLLM's documented order for + RL weight updates: wake weights, update, wake the KV cache. + """ + await self.engine.wake_up(tags=["weights"]) + self.kv_cache_awake = False + + async def wake_kv_cache(self) -> None: + if not self.kv_cache_awake: + await self.engine.wake_up(tags=["kv_cache"]) + self.kv_cache_awake = True async def sleep(self) -> None: await self.engine.sleep(level=1) + self.kv_cache_awake = False async def ensure_weights(self, weights_path: str | None) -> None: - """Make the engine hold weights_path; None keeps whatever it holds.""" + """Make the engine hold weights_path and be ready to generate; None keeps whatever it holds.""" if self.update_failed: raise RuntimeError("Weight update failed; restart the sampler before serving") - if weights_path is None or weights_path == self.weights_path: - return - # No finally/resume on failure: partially updated weights must not be served. - try: - await self.engine.pause_generation(mode="wait", clear_cache=True) - await self.engine.start_weight_update() - await self.engine.update_weights(WeightTransferUpdateRequest(update_info={"target_weights_path": weights_path})) - await self.engine.finish_weight_update(weight_version=weights_path) - await self.engine.reset_encoder_cache() - await self.engine.resume_generation() - except BaseException: - self.update_failed = True - raise - self.weights_path = weights_path + if weights_path is not None and weights_path != self.weights_path: + # No finally/resume on failure: partially updated weights must not be served. + try: + await self.engine.pause_generation(mode="wait", clear_cache=True) + await self.engine.start_weight_update() + await self.engine.update_weights(WeightTransferUpdateRequest(update_info={"target_weights_path": weights_path})) + await self.engine.finish_weight_update(weight_version=weights_path) + await self.engine.reset_encoder_cache() + await self.engine.resume_generation() + except BaseException: + self.update_failed = True + raise + self.weights_path = weights_path + await self.wake_kv_cache() async def generate(self, request: dict[str, Any]) -> dict[str, Any]: request_id = request["request_id"] diff --git a/tests/test_estimator.py b/tests/test_estimator.py index f8e79b43..64e38648 100644 --- a/tests/test_estimator.py +++ b/tests/test_estimator.py @@ -45,20 +45,18 @@ def test_sampler_matches_the_live_samplers(self) -> None: self.assertAlmostEqual(predicted_kv / GIB, kv, delta=0.3, msg=model) def test_fft_sampler_keeps_room_for_its_weight_update(self) -> None: - # gemma-4-e2b's sparse update stages its 4.38 GiB per-layer embedding - # densely on the device; sized without it the sampler landed on an L4 and - # died at the first update. The claim carries it, and still fits the L4 - # (so does vLLM's share once it is told to leave the room, see - # vllm_options.gpu_memory_utilization). LoRA samplers never update base - # weights and pay nothing. + # gemma-4-e2b's sparse update stages its 2.35 B-element per-layer embedding + # densely on the device, 11 GiB at the peak; sized without it the sampler + # landed on an L4 and died at the first update. With it the claim no longer + # fits a 24 GB device, so gemma FFT sampling belongs on the 80 GB tier. A + # 0.6B model's update is small and its FFT sampler still fits an L4. LoRA + # samplers never update base weights and pay nothing. room = weight_update_bytes(MODEL_TO_LARGEST_TENSOR_ELEMENTS["gemma-4-e2b"]) - self.assertGreater(room, 4 * GIB) - full = footprint("google/gemma-4-e2b", "full", "sampler").accelerator_bytes - self.assertGreater(full, 16 * GIB) - self.assertLess(full, 22 * GIB) - self.assertLess( - footprint("Qwen/Qwen3-0.6B", "full", "sampler").accelerator_bytes, footprint("Qwen/Qwen3-0.6B", "lora", "sampler").accelerator_bytes - ) + self.assertGreater(room, 11 * GIB) + self.assertGreater(footprint("google/gemma-4-e2b", "full", "sampler").accelerator_bytes, 22 * GIB) + self.assertLess(footprint("google/gemma-4-e2b", "full", "sampler").accelerator_bytes, 79 * GIB) + self.assertLess(footprint("Qwen/Qwen3-0.6B", "full", "sampler").accelerator_bytes, 22 * GIB) + self.assertEqual(weight_update_bytes(0), 0) def test_sampler_kv_cache_grows_with_the_model(self) -> None: # An 8B LoRA sampler sized at 22Gi landed on an L4 with 0.23 GiB of KV diff --git a/tests/test_sampler_patch.py b/tests/test_sampler_patch.py index e8d22cf9..bb776ae3 100644 --- a/tests/test_sampler_patch.py +++ b/tests/test_sampler_patch.py @@ -71,6 +71,25 @@ async def test_unchanged_weights_skip_update(self): self.engine.sleep.assert_not_called() self.engine.wake_up.assert_not_called() + async def test_kv_cache_wakes_after_the_weight_update(self): + # The update stages dense copies of the patched tensors; with the KV cache + # still parked that memory is free (gemma-4-e2b needs 11 GiB of it). + seen = [] + self.engine.wake_up.side_effect = lambda tags: seen.append(("wake", tuple(tags))) + self.engine.update_weights.side_effect = lambda req: seen.append(("update", req.update_info["target_weights_path"])) + await self.sampler.sleep() + await self.sampler.wake() + await process_batch(self.sampler, self.store, [{"request_id": "1", "weights_path": "a"}]) + self.assertEqual(seen, [("wake", ("weights",)), ("update", "a"), ("wake", ("kv_cache",))]) + # A second batch on the same weights neither updates nor wakes again. + await process_batch(self.sampler, self.store, [{"request_id": "2", "weights_path": "a"}]) + self.assertEqual(len(seen), 3) + # After the next sleep and wake the KV cache is brought back even with no update to do. + await self.sampler.sleep() + await self.sampler.wake() + await process_batch(self.sampler, self.store, [{"request_id": "3", "weights_path": "a"}]) + self.assertEqual(seen[3:], [("wake", ("weights",)), ("wake", ("kv_cache",))]) + async def test_failed_update_poisons_sampler_without_committing(self): self.engine.update_weights.side_effect = RuntimeError("invalid patch") await process_batch(self.sampler, self.store, [{"request_id": "bad", "weights_path": "a"}]) @@ -157,7 +176,9 @@ async def test_initialization_and_batches_own_slots_and_cleanup_once(self): await serve_time_sliced("test", self.store, self.factory) self.assertEqual(self.events, ["acquire", "create", "release", "acquire", "release"]) self.assertEqual(self.engine.sleep.await_count, 2) - self.engine.wake_up.assert_awaited_once() + # Weights come back when the slot is taken; the KV cache only once the + # weights are known to be current, so a weight update has the room. + self.assertEqual([c.kwargs["tags"] for c in self.engine.wake_up.await_args_list], [["weights"], ["kv_cache"]]) self.engine.shutdown.assert_called_once() self.slicer.register.assert_awaited_once_with(self.workload) self.slicer.unregister.assert_awaited_once_with(self.workload) diff --git a/tests/test_vllm_options.py b/tests/test_vllm_options.py index 0a26f7dc..400b31fa 100644 --- a/tests/test_vllm_options.py +++ b/tests/test_vllm_options.py @@ -63,6 +63,13 @@ def test_headroom_comes_off_vllms_share(self) -> None: self.assertAlmostEqual(h100, 0.90 - 4.88 / 80, places=3) self.assertLess((0.90 - l4) * 22.03, 4.88 + 0.01) + def test_the_share_still_holds_the_resident_weights(self) -> None: + # gemma-4-e2b on an L4: 11 GiB of headroom would leave 0.40 of 22 GiB, less + # than the 9 GiB of weights vLLM must load. The resident weights win. + with patch.dict(os.environ, {}, clear=True): + share = gpu_memory_utilization(headroom_bytes=12 * GIB, device_memory_bytes=22 * GIB, resident_bytes=9 * GIB) + self.assertAlmostEqual(share, 10 / 22, places=3) + def test_headroom_never_pushes_the_share_below_the_floor(self) -> None: with patch.dict(os.environ, {}, clear=True): self.assertEqual(gpu_memory_utilization(headroom_bytes=20 * GIB, device_memory_bytes=22 * GIB), vllm_options.MIN_GPU_MEMORY_UTILIZATION) @@ -99,8 +106,12 @@ def test_largest_tensor_is_read_from_the_headers(self) -> None: write_safetensors(os.path.join(d, "model-00001.safetensors"), {"embed": ("BF16", [1024, 64]), "norm": ("F32", [64])}) write_safetensors(os.path.join(d, "model-00002.safetensors"), {"head": ("BF16", [2048, 64])}) self.assertEqual(vllm_options.largest_checkpoint_tensor_bytes(d), 2048 * 64 * 2) + self.assertEqual(vllm_options.checkpoint_tensor_bytes(d), (2048 * 64 * 2, 1024 * 64 * 2 + 64 * 4 + 2048 * 64 * 2)) with patch.dict(os.environ, {}, clear=True): - self.assertEqual(vllm_options.weight_update_headroom_bytes(d), 2048 * 64 * 2 + vllm_options.WEIGHT_UPDATE_MARGIN_BYTES) + self.assertEqual( + vllm_options.weight_update_headroom_bytes(d), + int(2048 * 64 * 2 * vllm_options.WEIGHT_UPDATE_STAGING_FACTOR) + vllm_options.WEIGHT_UPDATE_MARGIN_BYTES, + ) def test_missing_checkpoint_means_no_headroom(self) -> None: with patch.dict(os.environ, {}, clear=True), patch.object(vllm_options, "local_snapshot", return_value=None): From 943eb383ec523ab0673d2f3ec123152d5b092113 Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 18:46:54 -0700 Subject: [PATCH 15/19] scheduler: cap the seats a shared claim may hold Host memory bounds how many workers park on a node, but says nothing about time: six FFT samplers on one H100 ran each step six times slower while other GPUs sat idle (runs/2026-10-17_demo_baselines, drill 3). --max-seats-per-claim (OPEN_RL_MAX_SEATS_PER_CLAIM, default 0 for no limit) is a hard policy limit. A ledger at the limit is not a selection candidate, and the ledger CAS loop refuses a join past it so two joiners on one snapshot cannot both take the last seat. A worker that finds every eligible ledger full cuts its own claim; with no free device it stays Pending, the reason names the limit, and it waits until a node arrives or a seat frees. Under binpack this fills one claim to the limit and moves to the next free device. --- scheduler/controller/cmd/manager/main.go | 6 +- .../controller/internal/controller/ledger.go | 5 ++ .../internal/controller/ledger_test.go | 24 +++++++ .../controller/workload_controller.go | 11 +++- .../controller/workload_controller_test.go | 63 +++++++++++++++++++ .../internal/placement/placement.go | 17 +++++ .../internal/placement/placement_test.go | 30 +++++++++ scheduler/docs/design.md | 37 ++++++++--- 8 files changed, 181 insertions(+), 12 deletions(-) diff --git a/scheduler/controller/cmd/manager/main.go b/scheduler/controller/cmd/manager/main.go index 6c734ab7..43d51dce 100644 --- a/scheduler/controller/cmd/manager/main.go +++ b/scheduler/controller/cmd/manager/main.go @@ -47,6 +47,7 @@ func main() { retryInterval time.Duration placementTimeout time.Duration strategy string + maxSeats int maxConcurrent int ) @@ -62,6 +63,8 @@ func main() { "How long a worker may go unplaced before the request is declared unsatisfiable. 0 waits forever.") flag.StringVar(&strategy, "placement-strategy", env("OPEN_RL_PLACEMENT_STRATEGY", string(placement.StrategyBinPack)), "binpack shares eligible FFT claims first; spread requests a GPU first, then falls back to FFT sharing. LoRA never shares.") + flag.IntVar(&maxSeats, "max-seats-per-claim", envInt("OPEN_RL_MAX_SEATS_PER_CLAIM", 0), + "Most workers that may share one claim; 0 is no limit. A worker that finds every eligible claim full cuts its own and waits Pending until a GPU is added or a seat frees.") flag.IntVar(&maxConcurrent, "max-concurrent-reconciles", envInt("OPEN_RL_MAX_CONCURRENT_RECONCILES", 4), "How many workers place at once. Seat booking is CAS-arbitrated, so concurrency risks only transient over-cut claims, which the sharing fallback drains.") @@ -116,6 +119,7 @@ func main() { RetryInterval: retryInterval, PlacementTimeout: placementTimeout, PlacementStrategy: parsedStrategy, + MaxSeatsPerClaim: maxSeats, MaxConcurrentReconciles: maxConcurrent, } if err := reconciler.SetupWithManager(mgr); err != nil { @@ -133,7 +137,7 @@ func main() { os.Exit(1) } - setupLog.Info("placing workers", "namespace", namespace, "deviceClass", deviceClass, "deviceDriver", deviceDriver, "strategy", strategy) + setupLog.Info("placing workers", "namespace", namespace, "deviceClass", deviceClass, "deviceDriver", deviceDriver, "strategy", strategy, "maxSeatsPerClaim", maxSeats) if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "manager exited") os.Exit(1) diff --git a/scheduler/controller/internal/controller/ledger.go b/scheduler/controller/internal/controller/ledger.go index 03e3dece..55d47a6d 100644 --- a/scheduler/controller/internal/controller/ledger.go +++ b/scheduler/controller/internal/controller/ledger.go @@ -97,6 +97,11 @@ func (r *WorkloadReconciler) ensureSeat(ctx context.Context, claimName string, s return nil, nil, errBookingContended } } + // The seat limit is re-checked here for the same reason: two joiners + // selecting from one snapshot must not both land the last seat. + if existing == nil && !createMissing && r.MaxSeatsPerClaim > 0 && len(claimLedger.Spec.Seats) >= r.MaxSeatsPerClaim { + return nil, nil, errBookingContended + } if existing != nil { *existing = seat } else { diff --git a/scheduler/controller/internal/controller/ledger_test.go b/scheduler/controller/internal/controller/ledger_test.go index 60341acf..28a4f9a4 100644 --- a/scheduler/controller/internal/controller/ledger_test.go +++ b/scheduler/controller/internal/controller/ledger_test.go @@ -123,6 +123,30 @@ func TestRebookingAdoptsTheRecordedSeat(t *testing.T) { } } +// The seat limit is enforced inside the ledger's CAS loop, so a joiner +// selecting from a snapshot that predates the last booking is refused. +func TestBookingRefusesASeatPastTheLimit(t *testing.T) { + r := newReconciler(t, append(enabledNode(), trainerWorker("w-a", "model-a"), trainerWorker("w-b", "model-b"), trainerWorker("w-c", "model-c"))...) + r.PlacementStrategy = placement.StrategyBinPack + r.MaxSeatsPerClaim = 2 + settle(t, r, "w-a") + claim := claimOf(t, r, "w-a") + allocateClaim(t, r, claim) + settle(t, r, "w-b") + if got := claimOf(t, r, "w-b"); got != claim { + t.Fatalf("w-b on %q, want %q", got, claim) + } + c := getWorker(t, r, "w-c") + if _, _, err := r.ensureSeat(context.Background(), claim, newSeat(c, requestFrom(c)), false); err != errBookingContended { + t.Fatalf("booking a third seat under a limit of two returned %v, want errBookingContended", err) + } + // The founder's own re-book is not a join and is never refused. + a := getWorker(t, r, "w-a") + if _, _, err := r.ensureSeat(context.Background(), claim, newSeat(a, requestFrom(a)), true); err != nil { + t.Fatalf("re-booking the founder: %v", err) + } +} + func TestSharingRechecksTheLedgerAfterSelection(t *testing.T) { resident, incoming := trainerWorker("resident", "model-a"), trainerWorker("incoming", "model-b") r := newReconciler(t, append(enabledNode(), resident, incoming)...) diff --git a/scheduler/controller/internal/controller/workload_controller.go b/scheduler/controller/internal/controller/workload_controller.go index d8dfe885..aa46f1f6 100644 --- a/scheduler/controller/internal/controller/workload_controller.go +++ b/scheduler/controller/internal/controller/workload_controller.go @@ -54,6 +54,13 @@ type WorkloadReconciler struct { // PlacementStrategy orders the two placement moves; the zero value reads // as binpack. PlacementStrategy placement.Strategy + // MaxSeatsPerClaim caps how many workers share one claim; zero means no + // limit. A worker that finds every eligible ledger at the limit cuts its + // own claim and, if no device is free, stays Pending until one is added or + // a seat frees. It bounds a time-sliced step at the limit times the + // single-worker time, and it turns a full fleet into standing claims an + // autoscaler can see. + MaxSeatsPerClaim int // MaxConcurrentReconciles is how many workers place at once. MaxConcurrentReconciles int @@ -109,7 +116,9 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, fmt.Errorf("cannot read the fleet, placing nothing this pass: %w", err) } - return r.place(ctx, &worker, requestFrom(&worker), fleet) + request := requestFrom(&worker) + request.MaxSeats = r.MaxSeatsPerClaim + return r.place(ctx, &worker, request, fleet) } type placementScope struct { diff --git a/scheduler/controller/internal/controller/workload_controller_test.go b/scheduler/controller/internal/controller/workload_controller_test.go index a9658d6d..bd9c4c9a 100644 --- a/scheduler/controller/internal/controller/workload_controller_test.go +++ b/scheduler/controller/internal/controller/workload_controller_test.go @@ -687,6 +687,69 @@ func TestReconcileBinPacksOntoExistingClaims(t *testing.T) { } } +// With a seat limit, binpack fills a claim to the limit, moves to a free +// device, and once every device is at the limit the next worker waits on a +// standing claim of its own until a seat frees. Two devices, limit two: a and +// b share the first, c and d the second, e stands pending; when b leaves, e +// takes its seat. +func TestReconcileStopsPackingAtTheSeatLimit(t *testing.T) { + r := newReconciler(t, append(enabledNode(), + trainerWorker("w-a", "model-a"), trainerWorker("w-b", "model-b"), trainerWorker("w-c", "model-c"), + trainerWorker("w-d", "model-d"), trainerWorker("w-e", "model-e"))...) + r.PlacementStrategy = placement.StrategyBinPack + r.MaxSeatsPerClaim = 2 + + settle(t, r, "w-a") + allocateClaim(t, r, claimOf(t, r, "w-a")) + settle(t, r, "w-b") + first := claimOf(t, r, "w-a") + if got := claimOf(t, r, "w-b"); got != first { + t.Fatalf("w-b landed on %q, want w-a's claim %q (under the limit)", got, first) + } + + settle(t, r, "w-c") + second := claimOf(t, r, "w-c") + if second == first { + t.Fatal("w-c joined a claim at the limit, want a claim of its own") + } + allocateClaim(t, r, second) + settle(t, r, "w-d") + if got := claimOf(t, r, "w-d"); got != second { + t.Fatalf("w-d landed on %q, want w-c's claim %q (under the limit)", got, second) + } + + // Both devices are claimed and both ledgers are full: w-e cuts a claim + // that cannot allocate, kube-scheduler declines its pod, and the sharing + // fallback finds no ledger under the limit. It stays Pending on its own + // claim with the limit in the reason. + settle(t, r, "w-e") + own := claimOf(t, r, "w-e") + if own == first || own == second { + t.Fatalf("w-e landed on %q, want a standing claim of its own", own) + } + fallBackToSharing(t, r, "w-e") + if got := claimOf(t, r, "w-e"); got != own { + t.Fatalf("w-e moved to %q past the limit, want it waiting on %q", got, own) + } + w := getWorker(t, r, "w-e") + if w.Status.Phase != openrlv1alpha1.PhasePending || !strings.Contains(w.Status.Reason, "at most 2 workers") { + t.Fatalf("w-e is %s %q, want Pending with the limit named", w.Status.Phase, w.Status.Reason) + } + + // A seat frees: w-b leaves the first claim and the next pass seats w-e. + if err := r.releaseSeatAndReclaim(context.Background(), first, "w-b"); err != nil { + t.Fatal(err) + } + fallBackToSharing(t, r, "w-e") + if got := claimOf(t, r, "w-e"); got != first { + t.Fatalf("w-e is on %q after a seat freed, want %q", got, first) + } + ledger := getLedger(t, r, ledgerNameFor(first)) + if len(ledger.Spec.Seats) != 2 { + t.Fatalf("first ledger holds %d seats, want 2: %+v", len(ledger.Spec.Seats), ledger.Spec.Seats) + } +} + // A trainer and a sampler on one accelerator, once the devices are contended. // Role selects which node pools may host a worker and nothing else -- in // particular it does not partition claims -- so on a node labelled for both, diff --git a/scheduler/controller/internal/placement/placement.go b/scheduler/controller/internal/placement/placement.go index 4cd5479e..4627c9c3 100644 --- a/scheduler/controller/internal/placement/placement.go +++ b/scheduler/controller/internal/placement/placement.go @@ -200,6 +200,17 @@ type Request struct { // HostRequestBytes is the pod template's memory request: what this // worker's pod asks the node for, resident or parked. HostRequestBytes int64 + // MaxSeats is the most workers a shared claim may hold once this one + // joins; zero means no limit. A ledger at the limit is not a candidate: + // the worker cuts its own claim, and if no device is free it waits. Set + // from the controller's flag, not from the spec: it is fleet policy. + MaxSeats int +} + +// AtSeatLimit reports whether a claim already holds as many workers as the +// request allows a shared claim to hold. +func (r Request) AtSeatLimit(claim *Claim) bool { + return r.MaxSeats > 0 && claim.Workers() >= r.MaxSeats } // OwnerKey is the fairness unit the timeslicer serves this worker under; a @@ -317,6 +328,9 @@ func SelectClaim(req Request, fleet *Fleet) *Claim { if req.Memory > node.DeviceMemoryBytes { continue } + if req.AtSeatLimit(claim) { + continue + } // Node-wide: every pod on the node draws from one allocatable pool. // A node reporting no memory skips the check rather than refusing all. if node.HostMemoryBytes > 0 && hostByNode[node.Name]+req.HostRequestBytes > node.HostMemoryBytes { @@ -359,6 +373,9 @@ func Explain(req Request, fleet *Fleet, detail string) string { case len(candidateNodes(req, fleet)) > 0: // The hardware exists; it is busy. Retrying is the right move. reason = "WaitingForCapacity: a pool fits this workload but none has a free seat or a free accelerator" + if req.MaxSeats > 0 { + reason += fmt.Sprintf("; shared claims hold at most %d workers", req.MaxSeats) + } default: biggest := pools[0] for _, node := range pools[1:] { diff --git a/scheduler/controller/internal/placement/placement_test.go b/scheduler/controller/internal/placement/placement_test.go index f1c3c515..e0c871ad 100644 --- a/scheduler/controller/internal/placement/placement_test.go +++ b/scheduler/controller/internal/placement/placement_test.go @@ -184,6 +184,36 @@ func TestSelectClaimPrefersFewestWorkersThenName(t *testing.T) { // Tight fit lives in the tier order: the tier wasting the least device // memory leads, so small work prefers small devices -- a preference DRA // honors when it can, never a survey of what is free. +func TestSelectClaimSkipsLedgersAtTheSeatLimit(t *testing.T) { + // Two shared ledgers on one node: one holds two workers, one holds one. + // With a limit of two the full ledger is not a candidate even though it + // would otherwise be preferred by name; with no limit it would still lose + // to the emptier one, so the test fills both to the limit as well. + fleet := NewFleet() + fleet.Nodes["n"] = &Node{Name: "n", DeviceCount: 2, DeviceMemoryBytes: 96 << 30, Roles: map[string]bool{"trainer": true}} + full := &Claim{Name: "a", Node: "n", DeviceCount: 1} + full.Book("w1", "o1", 0, true) + full.Book("w2", "o2", 0, true) + light := &Claim{Name: "b", Node: "n", DeviceCount: 1} + light.Book("w3", "o3", 0, true) + fleet.Claims["a"], fleet.Claims["b"] = full, light + + req := Request{Shareable: true, Role: "trainer", Memory: 24 << 30, WorkerID: "w4", MaxSeats: 2} + if got := SelectClaim(req, fleet); got == nil || got.Name != "b" { + t.Fatalf("selected %v, want the ledger under the limit", got) + } + light.Book("w4", "o4", 0, true) + if got := SelectClaim(req, fleet); got != nil { + t.Fatalf("selected %q with every ledger at the limit, want none", got.Name) + } + if got := SelectClaim(Request{Shareable: true, Role: "trainer", Memory: 24 << 30, WorkerID: "w5"}, fleet); got == nil { + t.Fatal("with no limit a full ledger is still joinable") + } + if reason := Explain(req, fleet, ""); !strings.Contains(reason, "at most 2 workers") { + t.Errorf("reason does not name the limit: %q", reason) + } +} + func TestTiersPreferTheTightestFit(t *testing.T) { fleet := NewFleet() fleet.Nodes["l4"] = bigNode("l4", 4, 24, "trainer") diff --git a/scheduler/docs/design.md b/scheduler/docs/design.md index b86bbbb2..8d052379 100644 --- a/scheduler/docs/design.md +++ b/scheduler/docs/design.md @@ -513,15 +513,29 @@ memory are read from its owned `ResourceClaim` and the DRA inventory. ### Seat limit -There is no per-claim seat count. Host memory is the limit on parked -workers: every seat costs host memory even while its worker is off the GPU --- a suspended worker parks its state in node RAM -- so admission sums the -seats' pod requests against the node's allocatable memory (section 6), and -kube-scheduler enforces those same requests for real when the pod binds. -The two checks split the roles a seat count used to conflate: the advisory -sum steers workers away from full nodes, and the bind is the hard cap. A -join that slips past the advisory check on a stale snapshot produces an -unschedulable pod, which the wedged-claim path unwinds. +Host memory is the natural limit on parked workers: every seat costs host +memory even while its worker is off the GPU -- a suspended worker parks its +state in node RAM -- so admission sums the seats' pod requests against the +node's allocatable memory (section 6), and kube-scheduler enforces those +same requests for real when the pod binds. The two checks split the roles a +seat count used to conflate: the advisory sum steers workers away from full +nodes, and the bind is the hard cap. A join that slips past the advisory +check on a stale snapshot produces an unschedulable pod, which the +wedged-claim path unwinds. + +Host memory says nothing about time, and a time-sliced claim with six seats +runs each worker's step six times slower. `--max-seats-per-claim` +(`OPEN_RL_MAX_SEATS_PER_CLAIM`, default 0 for no limit) is the policy limit +on top: a ledger holding that many workers is not a candidate for selection, +and the ledger CAS loop refuses a join that would exceed it, so two joiners +working from one snapshot cannot both take the last seat. A worker that +finds every eligible ledger at the limit cuts a claim of its own; if no +device is free the claim stands pending, the status reason names the +limit, and the worker waits until a node arrives or a seat frees. It is +deliberately a hard limit rather than a preference: the bound on step time +holds, and a full fleet shows as standing claims an autoscaler can act on. +The limit is fleet policy and lives on the controller, not on nodes or +workloads; a per-device-tier form is a natural extension. An unallocated ledger contains only its dedicated workload and is not joinable: its node is unknown, so nothing about it can be checked against @@ -786,7 +800,10 @@ the dedicated-first arc described above. before cutting a claim at all; only a workload no ledger can seat cuts a dedicated claim. FFT can share; LoRA always cuts its own claim. Eligibility and preference are the shared-ledger rules below, unchanged; a booking that - loses the last seat to a race falls through to a dedicated claim. + loses the last seat to a race falls through to a dedicated claim. With + `--max-seats-per-claim` set, a ledger at the limit is not eligible, so + binpack fills one claim to the limit and then moves to the next free + device (section 4, "Seat limit"). One more placement policy is recorded as future work, not V1 behavior: From fcf8b8da195449b1fb2c24e322203892b705ec75 Mon Sep 17 00:00:00 2001 From: droot Date: Sat, 17 Oct 2026 19:23:18 -0700 Subject: [PATCH 16/19] scripts: let demo Jobs carry openrl.* session tags --lora-tags and --fft-tags set TINKER_TAGS on the client, e.g. openrl.sampler_replicas=2, so a drill can give the shared LoRA runtimes a second sampler without touching the recipe. --- scripts/demo_jobs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/demo_jobs.py b/scripts/demo_jobs.py index 4cf12c74..a48edb8d 100644 --- a/scripts/demo_jobs.py +++ b/scripts/demo_jobs.py @@ -60,6 +60,10 @@ env: - name: PYTHONUNBUFFERED value: "1" + # openrl.* session settings, e.g. openrl.sampler_replicas=2; the SDK + # sends TINKER_TAGS as the session's tags and the API server reads them. + - name: TINKER_TAGS + value: "{tags}" - name: HF_HOME value: "/hf" resources: @@ -123,6 +127,8 @@ def main() -> None: parser.add_argument("--groups-per-batch", type=int, default=8) parser.add_argument("--max-tokens", type=int, default=512) parser.add_argument("--namespace", default="openrl-system") + parser.add_argument("--lora-tags", default="", help="TINKER_TAGS for the LoRA jobs, e.g. openrl.sampler_replicas=2") + parser.add_argument("--fft-tags", default="", help="TINKER_TAGS for the FFT jobs") args = parser.parse_args() out = [] for spec in args.job: @@ -146,6 +152,7 @@ def main() -> None: group_size=args.group_size, groups_per_batch=args.groups_per_batch, max_tokens=args.max_tokens, + tags=args.lora_tags if kind == "lora" else args.fft_tags, ) ) sys.stdout.write("".join(out)) From aba8144d5ef15c89da1ce96a4d69883411ac3b10 Mon Sep 17 00:00:00 2001 From: droot Date: Sun, 18 Oct 2026 23:56:11 -0700 Subject: [PATCH 17/19] scheduler: show the status reason in kubectl get workloads A waiting worker's reason, for example the seat limit, was only visible in describe. It is the column to watch during a drill. --- scheduler/controller/api/v1alpha1/workload_types.go | 1 + scheduler/deploy/base/00-workload-crd.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/scheduler/controller/api/v1alpha1/workload_types.go b/scheduler/controller/api/v1alpha1/workload_types.go index 57c3d8cb..fb7329c7 100644 --- a/scheduler/controller/api/v1alpha1/workload_types.go +++ b/scheduler/controller/api/v1alpha1/workload_types.go @@ -193,6 +193,7 @@ type WorkloadStatus struct { // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` // +kubebuilder:printcolumn:name="Claim",type=string,JSONPath=`.status.claimName` // +kubebuilder:printcolumn:name="Node",type=string,JSONPath=`.status.nodeName` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.reason` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // Workload is the scheduling request for a single worker process. The diff --git a/scheduler/deploy/base/00-workload-crd.yaml b/scheduler/deploy/base/00-workload-crd.yaml index 63edb92e..4154e6cf 100644 --- a/scheduler/deploy/base/00-workload-crd.yaml +++ b/scheduler/deploy/base/00-workload-crd.yaml @@ -38,6 +38,9 @@ spec: - jsonPath: .status.nodeName name: Node type: string + - jsonPath: .status.reason + name: Reason + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date From fc1ccbbead7b2b109673d5664c975ca33df65d83 Mon Sep 17 00:00:00 2001 From: droot Date: Sun, 18 Oct 2026 23:56:28 -0700 Subject: [PATCH 18/19] scripts: drill cells in any region, and a one-command rehearsal cell_up.sh brings up an OpenRL cell (cluster, Spot GPU pool, drivers, bundle) in a chosen region, reports the pool's quota or stockout errors instead of hanging, and leaves the caller's kubectl context alone; cell_seed_hf.sh copies the model cache in through a bucket; cell_down.sh removes the cell. demo_rehearsal.sh checks the fleet is idle, trims the shared volume, optionally warms the LoRA runtimes, kicks off the job set and watches. --- scripts/cell_down.sh | 16 ++++++ scripts/cell_seed_hf.sh | 68 ++++++++++++++++++++++++ scripts/cell_up.sh | 105 ++++++++++++++++++++++++++++++++++++++ scripts/demo_rehearsal.sh | 81 +++++++++++++++++++++++++++++ 4 files changed, 270 insertions(+) create mode 100755 scripts/cell_down.sh create mode 100755 scripts/cell_seed_hf.sh create mode 100755 scripts/cell_up.sh create mode 100755 scripts/demo_rehearsal.sh diff --git a/scripts/cell_down.sh b/scripts/cell_down.sh new file mode 100755 index 00000000..6836e503 --- /dev/null +++ b/scripts/cell_down.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tear down a drill cell: the OpenRL namespace first (so the Filestore +# instance behind open-rl-shared-pvc is released), then the cluster. +# +# scripts/cell_down.sh +set -euo pipefail +NAME=${1:?name}; REGION=${2:?region} +PROJECT=${PROJECT:-cdrollouts-sunilarora} +CTX="gke_${PROJECT}_${REGION}_${NAME}" +if kubectl --context "$CTX" get ns openrl-system >/dev/null 2>&1; then + kubectl --context "$CTX" delete ns openrl-system --wait=true --timeout=10m || true +fi +gcloud container clusters delete "$NAME" --location "$REGION" --project "$PROJECT" --quiet +kubectl config delete-context "$CTX" >/dev/null 2>&1 || true +echo "cell $NAME removed. Check for a leftover Filestore instance:" +echo " gcloud filestore instances list --project $PROJECT --filter=\"zone~$REGION\"" diff --git a/scripts/cell_seed_hf.sh b/scripts/cell_seed_hf.sh new file mode 100755 index 00000000..ce025b9b --- /dev/null +++ b/scripts/cell_seed_hf.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Copy the Hugging Face cache (models and datasets) from the main cluster's +# shared volume into a drill cell's volume, through a GCS bucket. The cell's +# nodes can read GCS (devstorage.read_only) but not write it, so the upload +# streams through this machine; the download runs inside the cell. +# +# scripts/cell_seed_hf.sh [source context] [models...] +# +# Defaults: source gke_cdrollouts-sunilarora_us-central1_open-rl-dra; models +# google/gemma-4-e2b and Qwen/Qwen3-0.6B plus the gsm8k dataset. Reruns skip +# the upload when the archive is already in the bucket. +set -euo pipefail +DST=${1:?cell context} +SRC=${2:-gke_cdrollouts-sunilarora_us-central1_open-rl-dra} +shift 2 2>/dev/null || shift $# +MODELS=${*:-"google/gemma-4-e2b Qwen/Qwen3-0.6B"} +PROJECT=${PROJECT:-cdrollouts-sunilarora} +BUCKET=${BUCKET:-gs://$PROJECT-openrl-cache} +HF=/mnt/shared/open-rl/huggingface +cd "$(dirname "$0")/.." + +gcloud storage buckets describe "$BUCKET" --project "$PROJECT" >/dev/null 2>&1 || gcloud storage buckets create "$BUCKET" --project "$PROJECT" --location US --uniform-bucket-level-access +paths="datasets hub/datasets--openai--gsm8k" +for m in $MODELS; do paths="$paths hub/models--${m//\//--}"; done + +archive="$BUCKET/hf-cache.tgz" +if gcloud storage ls "$archive" >/dev/null 2>&1; then + echo "archive exists: $archive (delete it to re-upload)" +else + echo "streaming $paths from $SRC to $archive" + # shellcheck disable=SC2086 + kubectl --context "$SRC" -n openrl-system exec deploy/open-rl-api-server -- sh -c "cd $HF && tar czf - $paths" \ + | gcloud storage cp - "$archive" +fi + +echo "unpacking inside $DST" +kubectl --context "$DST" -n openrl-system delete job hf-seed --ignore-not-found >/dev/null +cat </dev/null +apiVersion: batch/v1 +kind: Job +metadata: + name: hf-seed +spec: + backoffLimit: 1 + ttlSecondsAfterFinished: 3600 + template: + spec: + restartPolicy: Never + containers: + - name: seed + image: google/cloud-sdk:slim + command: ["sh", "-c"] + args: + - | + set -e + mkdir -p $HF && cd $HF + gcloud storage cp $archive - | tar xzf - + du -sh $HF/hub/* $HF/datasets 2>/dev/null + volumeMounts: + - name: shared + mountPath: /mnt/shared + volumes: + - name: shared + persistentVolumeClaim: + claimName: open-rl-shared-pvc +EOF +kubectl --context "$DST" -n openrl-system wait --for=condition=complete job/hf-seed --timeout=60m +kubectl --context "$DST" -n openrl-system logs job/hf-seed | tail -8 diff --git a/scripts/cell_up.sh b/scripts/cell_up.sh new file mode 100755 index 00000000..e95f0ba2 --- /dev/null +++ b/scripts/cell_up.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Bring up an OpenRL drill cell in any GCP region: a GKE cluster, one GPU node +# pool (Spot by default), the NVIDIA driver and DRA driver, and the OpenRL +# bundle from k8s/deploy/gke-cell. Trainer, sampler, shared storage and the +# client Jobs all live in this region, which is what locality needs. +# +# scripts/cell_up.sh [machine] [accelerator] [count] [local-ssds] +# +# scripts/cell_up.sh cell-use4 us-east4 us-east4-b # 1 x a3-ultragpu-8g (8 x H200), Spot +# scripts/cell_up.sh cell-use1 us-east1 us-east1-b a4-highgpu-8g nvidia-b200 8 32 # 1 x a4-highgpu-8g (8 x B200), Spot +# SPOT=0 scripts/cell_up.sh ... # on-demand (the demo, with a reservation) +# GPU_NODES=2 scripts/cell_up.sh ... # more than one GPU node +# +# Idempotent: every step checks for what already exists, so a run that stopped +# at a quota or stockout error can be rerun as is. The GPU pool is created +# asynchronously; the script waits for a node and reports the pool's errors +# (GCE_QUOTA_EXCEEDED, ZONE_RESOURCE_POOL_EXHAUSTED) instead of hanging. +# +# After it finishes: scripts/cell_seed_hf.sh copies the model cache from the +# main cluster, then the warm-up round and the drills run against this cell's +# kubectl context (printed at the end). +set -euo pipefail +NAME=${1:?name}; REGION=${2:?region}; ZONE=${3:?zone} +MACHINE=${4:-a3-ultragpu-8g}; ACCEL=${5:-nvidia-h200-141gb}; COUNT=${6:-8}; SSDS=${7:-32} +PROJECT=${PROJECT:-cdrollouts-sunilarora} +SPOT=${SPOT:-1} +GPU_NODES=${GPU_NODES:-1} +OVERLAY=${OVERLAY:-k8s/deploy/gke-cell} +cd "$(dirname "$0")/.." +CTX="gke_${PROJECT}_${REGION}_${NAME}" +log() { printf '\n== %s\n' "$*"; } + +log "cluster $NAME in $REGION ($ZONE)" +if ! gcloud container clusters describe "$NAME" --location "$REGION" --project "$PROJECT" >/dev/null 2>&1; then + gcloud container clusters create "$NAME" --location "$REGION" --node-locations "$ZONE" --project "$PROJECT" \ + --release-channel regular --machine-type e2-standard-4 --num-nodes 1 --disk-size 100 \ + --addons GcpFilestoreCsiDriver,GcePersistentDiskCsiDriver \ + --logging SYSTEM,WORKLOAD --monitoring SYSTEM --quiet +fi +# get-credentials switches the current context; put it back so other shells keep theirs. +prev_ctx=$(kubectl config current-context 2>/dev/null || true) +gcloud container clusters get-credentials "$NAME" --location "$REGION" --project "$PROJECT" >/dev/null +[ -n "$prev_ctx" ] && kubectl config use-context "$prev_ctx" >/dev/null +K="kubectl --context $CTX" + +log "GPU node pool gpu-8x ($MACHINE, $COUNT x $ACCEL, spot=$SPOT)" +if ! gcloud container node-pools describe gpu-8x --cluster "$NAME" --location "$REGION" --project "$PROJECT" >/dev/null 2>&1; then + spot_flag=""; [ "$SPOT" = 1 ] && spot_flag="--spot" + disk_flags="--disk-type hyperdisk-balanced --disk-size 200 --ephemeral-storage-local-ssd count=$SSDS" + case "$MACHINE" in a3-highgpu-*|g2-*) disk_flags="--disk-type pd-balanced --disk-size 200" ;; esac + # shellcheck disable=SC2086 + gcloud container node-pools create gpu-8x --cluster "$NAME" --location "$REGION" --node-locations "$ZONE" --project "$PROJECT" \ + --machine-type "$MACHINE" --accelerator "type=$ACCEL,count=$COUNT,gpu-driver-version=disabled" \ + --num-nodes "$GPU_NODES" --image-type COS_CONTAINERD $disk_flags $spot_flag \ + --node-labels openrl.io/enabled=true,openrl.io/trainer=true,openrl.io/sampler=true,nvidia.com/gpu.present=true,gke-no-default-nvidia-gpu-device-plugin=true \ + --node-taints nvidia.com/gpu=present:NoSchedule --async --quiet +fi + +log "waiting for a GPU node (pool errors are reported below)" +mig="" +for _ in $(seq 1 60); do + if [ "$($K get nodes -l cloud.google.com/gke-nodepool=gpu-8x --no-headers 2>/dev/null | grep -c ' Ready')" -ge 1 ]; then break; fi + [ -z "$mig" ] && mig=$(gcloud compute instance-groups managed list --project "$PROJECT" --filter="name~gke-$NAME-gpu-8x" --format='value(name)' 2>/dev/null | head -1 || true) + if [ -n "$mig" ]; then + err=$(gcloud compute instance-groups managed list-errors "$mig" --zone "$ZONE" --project "$PROJECT" --format='value(error.code,error.message)' 2>/dev/null | head -1 || true) + [ -n "$err" ] && echo " pool: $err" | cut -c1-200 + fi + sleep 20 +done +if [ "$($K get nodes -l cloud.google.com/gke-nodepool=gpu-8x --no-headers 2>/dev/null | grep -c ' Ready')" -lt 1 ]; then + echo "no GPU node after 20 minutes; the pool keeps retrying. Rerun this script when capacity or quota arrives." >&2 + exit 2 +fi + +log "NVIDIA kernel driver (installer DaemonSet) and DRA driver (Helm)" +$K apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded-latest.yaml >/dev/null +helm repo add nvidia https://helm.ngc.nvidia.com/nvidia >/dev/null 2>&1 || true +helm repo update >/dev/null +helm --kube-context "$CTX" upgrade --install nvidia-dra-driver-gpu nvidia/nvidia-dra-driver-gpu \ + --version 25.8.0 --create-namespace --namespace nvidia-dra-driver-gpu \ + --set nvidiaDriverRoot=/home/kubernetes/bin/nvidia/ >/dev/null +for _ in $(seq 1 60); do + n=$($K get resourceslices -o json 2>/dev/null | python3 -c 'import json,sys; print(sum(len(s["spec"].get("devices",[])) for s in json.load(sys.stdin)["items"]))' 2>/dev/null || echo 0) + [ "$n" -ge "$COUNT" ] && break + sleep 10 +done +echo " ResourceSlice devices: $n" + +log "OpenRL bundle from $OVERLAY" +$K create namespace openrl-system --dry-run=client -o yaml | $K apply -f - >/dev/null +$K apply --server-side --force-conflicts -k "$OVERLAY" >/dev/null +$K -n openrl-system rollout status deployment/open-rl-api-server --timeout=10m +$K -n openrl-system rollout status deployment/open-rl-scheduler --timeout=5m +$K -n openrl-system rollout status daemonset/open-rl-worker-prepull --timeout=15m || true + +log "cell $NAME is up" +$K get nodes -L cloud.google.com/gke-accelerator,openrl.io/trainer,openrl.io/sampler +$K -n openrl-system get pods +cat < [--warm] [--jobs " ..."] [--steps N] [--context CTX] +# +# scripts/demo_rehearsal.sh rehearsal-1 --warm +# scripts/demo_rehearsal.sh big-1 --jobs "lora:Qwen/Qwen3-0.6B:6 fft:Qwen/Qwen3-0.6B:6 lora:google/gemma-4-e2b:6 fft:google/gemma-4-e2b:6" +# +# --warm starts one short job per LoRA base model and launches the set as soon +# as those finish, while the runtimes are still up (the API server reaps a +# runtime about two minutes after its last session ends). The warm-up jobs are +# labeled -warm so the report can tell them apart. +# +# Storage: finished FFT jobs leave sampler weights and checkpoints behind, 2-10 +# GB each. Entries under sampler_full and checkpoints older than RETAIN_MIN +# minutes (default 180) are removed before the run. +set -euo pipefail +RUN=${1:?run name}; shift +WARM=0; STEPS=3; CTX=${KCTX:-$(kubectl config current-context)} +JOBS="lora:Qwen/Qwen3-0.6B:3 fft:Qwen/Qwen3-0.6B:4 lora:google/gemma-4-e2b:3 fft:google/gemma-4-e2b:2" +while [ $# -gt 0 ]; do + case $1 in + --warm) WARM=1 ;; + --jobs) JOBS=$2; shift ;; + --steps) STEPS=$2; shift ;; + --context) CTX=$2; shift ;; + *) echo "unknown argument $1" >&2; exit 2 ;; + esac + shift +done +IMAGE=${IMAGE:-us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-client:demo1-18409bd} +NS=${OPENRL_NAMESPACE:-openrl-system} +RETAIN_MIN=${RETAIN_MIN:-180} +cd "$(dirname "$0")/.." +K="kubectl --context $CTX -n $NS" +log() { printf '\n== %s %s\n' "$(date -u +%T)" "$*"; } + +log "fleet on $CTX" +kubectl --context "$CTX" get nodes -l openrl.io/enabled=true -L cloud.google.com/gke-accelerator,openrl.io/trainer,openrl.io/sampler --no-headers | awk '{print " " $1, $6, $7, $8}' | sed 's/gke-open-rl-dra-//' +busy=$($K get workloads --no-headers 2>/dev/null | wc -l | tr -d ' ') +if [ "$busy" != 0 ]; then + echo " $busy workloads are still running; wait for the fleet to be idle:" >&2 + $K get workloads >&2 + exit 1 +fi + +log "shared volume" +$K exec deploy/open-rl-api-server -- sh -c " + for d in /mnt/shared/open-rl/sampler_full /mnt/shared/open-rl/checkpoints; do + [ -d \$d ] && find \$d -mindepth 1 -maxdepth 1 -mmin +$RETAIN_MIN -exec rm -rf {} + 2>/dev/null + done + df -h /mnt/shared | tail -1" + +jobargs=() +for j in $JOBS; do jobargs+=(--job "$j"); done + +if [ "$WARM" = 1 ]; then + log "warm-up: one short job per LoRA base model" + warmargs=() + for j in $JOBS; do + kind=${j%%:*}; rest=${j#*:}; model=${rest%:*} + [ "$kind" = lora ] && warmargs+=(--job "lora:$model:1") + done + if [ ${#warmargs[@]} -gt 0 ]; then + python3 scripts/demo_jobs.py --image "$IMAGE" --run "$RUN-warm" "${warmargs[@]}" --steps 1 | $K apply -f - >/dev/null + for _ in $(seq 1 60); do + left=$($K get jobs -l "openrl.io/demo-run=$RUN-warm" -o json | python3 -c 'import json,sys; j=json.load(sys.stdin)["items"]; print(sum(1 for x in j if not (x["status"].get("succeeded") or x["status"].get("failed"))))') + [ "$left" = 0 ] && break + sleep 10 + done + echo " runtimes up:"; $K get workloads --no-headers | awk '{print " " $1, $2}' | grep lora || true + fi +fi + +log "kickoff $RUN: $JOBS, $STEPS steps" +python3 scripts/demo_jobs.py --image "$IMAGE" --run "$RUN" "${jobargs[@]}" --steps "$STEPS" | $K apply -f - | grep -c created +echo " watch: python3 scripts/demo_watch.py --run $RUN" +echo " report: python3 scripts/demo_report.py --run $RUN --steps $STEPS --budget 180" +exec python3 scripts/demo_watch.py --run "$RUN" --namespace "$NS" From 4272cf328afcebcc24d368395f3560bc478a3bc1 Mon Sep 17 00:00:00 2001 From: droot Date: Mon, 19 Oct 2026 11:57:02 -0700 Subject: [PATCH 19/19] runs: keep the written reports, drop the raw logs; add the fleet demo notes The cold-start archive kept 243 raw files (pod logs, event dumps, fio output) next to its report. The reports and timelines stay; the raw files go, they are reproducible with the scripts. Adds the fleet demo notes (drills 1 to 7 with the scorecard and what each change did) and the fleet request with the quota findings. --- .../A-0.5b-l4-cold.run.log | 34 - .../A-0.5b-l4-cold/api-server.log | 71 - .../A-0.5b-l4-cold/client.jsonl | 11 - .../A-0.5b-l4-cold/client.log | 12 - .../A-0.5b-l4-cold/collect-started.txt | 1 - .../A-0.5b-l4-cold/events.json | 661 -- ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 - ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 46 - .../A-0.5b-l4-cold/monitoring.json | 329 - .../A-0.5b-l4-cold/node-images-after.json | 48 - .../A-0.5b-l4-cold/node-images-before.json | 50 - .../A-0.5b-l4-cold/node-labels-during.txt | 8 - .../A-0.5b-l4-cold/pods.json | 6782 ---------------- .../A-0.5b-l4-cold/redis-ready.log | 299 - .../A-0.5b-l4-cold/scheduler.log | 2 - .../A-0.5b-l4-cold/timeline.json | 275 - .../A-0.5b-l4-cold/watch-started.txt | 1 - .../A-0.5b-l4-cold/worker-image.txt | 1 - .../A-0.5b-l4-cold/workloads.json | 355 - .../B-0.5b-l4-cached/api-server.log | 31 - .../B-0.5b-l4-cached/client.jsonl | 11 - .../B-0.5b-l4-cached/client.log | 16 - .../B-0.5b-l4-cached/collect-started.txt | 1 - .../B-0.5b-l4-cached/events.json | 269 - ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 - ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 47 - .../B-0.5b-l4-cached/monitoring.json | 212 - .../B-0.5b-l4-cached/node-images-after.json | 50 - .../B-0.5b-l4-cached/node-images-before.json | 50 - .../B-0.5b-l4-cached/pods.json | 6782 ---------------- .../B-0.5b-l4-cached/redis-ready.log | 136 - .../B-0.5b-l4-cached/scheduler.log | 2 - .../B-0.5b-l4-cached/timeline.json | 267 - .../B-0.5b-l4-cached/watch-started.txt | 1 - .../B-0.5b-l4-cached/worker-image.txt | 1 - .../B-0.5b-l4-cached/workloads.json | 355 - .../C-8b-h100-cold.run.log | 34 - .../C-8b-h100-cold/api-server.log | 176 - .../C-8b-h100-cold/client.jsonl | 11 - .../C-8b-h100-cold/client.log | 12 - .../C-8b-h100-cold/collect-started.txt | 1 - .../C-8b-h100-cold/events.json | 1053 --- .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 223 - .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 220 - .../C-8b-h100-cold/monitoring.json | 725 -- .../C-8b-h100-cold/node-images-after.json | 50 - .../C-8b-h100-cold/node-images-before.json | 48 - .../C-8b-h100-cold/node-labels-during.txt | 8 - .../C-8b-h100-cold/pods.json | 6782 ---------------- .../C-8b-h100-cold/redis-ready.log | 1734 ----- .../sampler-mountstats-sample1.txt | 2 - .../sampler-mountstats-sample2.txt | 2 - .../C-8b-h100-cold/scheduler.log | 2 - .../C-8b-h100-cold/timeline.json | 275 - .../trainer-mountstats-sample1.txt | 14 - .../trainer-mountstats-sample2.txt | 2 - .../C-8b-h100-cold/watch-started.txt | 1 - .../C-8b-h100-cold/worker-image.txt | 1 - .../C-8b-h100-cold/workloads.json | 355 - .../D-8b-h100-cached-interrupted/NOTE.txt | 1 - .../api-server.log | 38 - .../D-8b-h100-cached-interrupted/client.jsonl | 4 - .../D-8b-h100-cached-interrupted/client.log | 6 - .../collect-started.txt | 1 - .../D-8b-h100-cached-interrupted/events.json | 401 - .../node-images-after.json | 50 - .../node-images-before.json | 50 - .../node-labels-during.txt | 8 - .../D-8b-h100-cached-interrupted/pods.json | 5980 --------------- .../redis-ready.log | 1001 --- .../D-8b-h100-cached-interrupted/run.log | 36 - .../scheduler.log | 2 - .../timeline.json | 26 - .../watch-started.txt | 1 - .../worker-image.txt | 1 - .../workloads.json | 8 - .../D-8b-h100-cached.run.log | 35 - .../D-8b-h100-cached/api-server.log | 36 - .../D-8b-h100-cached/client.jsonl | 11 - .../D-8b-h100-cached/client.log | 12 - .../D-8b-h100-cached/collect-started.txt | 1 - .../D-8b-h100-cached/events.json | 661 -- .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 223 - .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 166 - .../D-8b-h100-cached/monitoring.json | 194 - .../D-8b-h100-cached/node-images-after.json | 50 - .../D-8b-h100-cached/node-images-before.json | 50 - .../D-8b-h100-cached/node-labels-during.txt | 8 - .../D-8b-h100-cached/pods.json | 6782 ---------------- .../D-8b-h100-cached/redis-ready.log | 147 - .../D-8b-h100-cached/scheduler.log | 2 - .../D-8b-h100-cached/timeline.json | 267 - .../D-8b-h100-cached/watch-started.txt | 1 - .../D-8b-h100-cached/worker-image.txt | 1 - .../D-8b-h100-cached/workloads.json | 355 - .../after-A1-0.5b-l4-cold.run.log | 35 - .../after-A1-0.5b-l4-cold/api-server.log | 75 - .../after-A1-0.5b-l4-cold/client.jsonl | 11 - .../after-A1-0.5b-l4-cold/client.log | 12 - .../after-A1-0.5b-l4-cold/collect-started.txt | 1 - .../after-A1-0.5b-l4-cold/events.json | 1311 ---- ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 - ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 34 - .../node-images-after.json | 49 - .../node-images-before.json | 51 - .../node-labels-during.txt | 8 - .../after-A1-0.5b-l4-cold/pods.json | 6782 ---------------- .../after-A1-0.5b-l4-cold/redis-ready.log | 309 - .../after-A1-0.5b-l4-cold/scheduler.log | 2 - .../after-A1-0.5b-l4-cold/timeline.json | 275 - .../after-A1-0.5b-l4-cold/watch-started.txt | 1 - .../after-A1-0.5b-l4-cold/worker-image.txt | 1 - .../after-A1-0.5b-l4-cold/workloads.json | 355 - .../after-B1-0.5b-l4-cached.run.log | 35 - .../after-B1-0.5b-l4-cached/api-server.log | 33 - .../after-B1-0.5b-l4-cached/client.jsonl | 11 - .../after-B1-0.5b-l4-cached/client.log | 12 - .../collect-started.txt | 1 - .../after-B1-0.5b-l4-cached/events.json | 1699 ----- ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 119 - ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 34 - .../node-images-after.json | 49 - .../node-images-before.json | 49 - .../node-labels-during.txt | 8 - .../after-B1-0.5b-l4-cached/pods.json | 6786 ----------------- .../after-B1-0.5b-l4-cached/redis-ready.log | 175 - .../after-B1-0.5b-l4-cached/scheduler.log | 2 - .../after-B1-0.5b-l4-cached/timeline.json | 275 - .../after-B1-0.5b-l4-cached/watch-started.txt | 1 - .../after-B1-0.5b-l4-cached/worker-image.txt | 1 - .../after-B1-0.5b-l4-cached/workloads.json | 355 - .../after-B2-0.5b-l4-cached.run.log | 35 - .../after-B2-0.5b-l4-cached/api-server.log | 30 - .../after-B2-0.5b-l4-cached/client.jsonl | 11 - .../after-B2-0.5b-l4-cached/client.log | 12 - .../collect-started.txt | 1 - .../after-B2-0.5b-l4-cached/events.json | 2025 ----- ...g-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt | 120 - ...g-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt | 34 - .../node-images-after.json | 49 - .../node-images-before.json | 49 - .../node-labels-during.txt | 8 - .../after-B2-0.5b-l4-cached/pods.json | 6782 ---------------- .../after-B2-0.5b-l4-cached/redis-ready.log | 153 - .../after-B2-0.5b-l4-cached/scheduler.log | 2 - .../after-B2-0.5b-l4-cached/timeline.json | 267 - .../after-B2-0.5b-l4-cached/watch-started.txt | 1 - .../after-B2-0.5b-l4-cached/worker-image.txt | 1 - .../after-B2-0.5b-l4-cached/workloads.json | 355 - .../after-C1-8b-h100-cold.run.log | 35 - .../after-C1-8b-h100-cold/api-server.log | 56 - .../after-C1-8b-h100-cold/client.jsonl | 11 - .../after-C1-8b-h100-cold/client.log | 12 - .../after-C1-8b-h100-cold/collect-started.txt | 1 - .../after-C1-8b-h100-cold/events.json | 2417 ------ .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 223 - .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 50 - .../after-C1-8b-h100-cold/monitoring.json | 320 - .../node-images-after.json | 51 - .../node-images-before.json | 49 - .../node-labels-during.txt | 8 - .../after-C1-8b-h100-cold/pods.json | 6782 ---------------- .../after-C1-8b-h100-cold/redis-ready.log | 575 -- .../after-C1-8b-h100-cold/scheduler.log | 2 - .../after-C1-8b-h100-cold/timeline.json | 275 - .../after-C1-8b-h100-cold/watch-started.txt | 1 - .../after-C1-8b-h100-cold/worker-image.txt | 1 - .../after-C1-8b-h100-cold/workloads.json | 355 - .../after-D1-8b-h100-cached.run.log | 35 - .../after-D1-8b-h100-cached/api-server.log | 32 - .../after-D1-8b-h100-cached/client.jsonl | 11 - .../after-D1-8b-h100-cached/client.log | 12 - .../collect-started.txt | 1 - .../after-D1-8b-h100-cached/events.json | 2743 ------- .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 222 - .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 50 - .../node-images-after.json | 51 - .../node-images-before.json | 51 - .../node-labels-during.txt | 8 - .../after-D1-8b-h100-cached/pods.json | 6782 ---------------- .../after-D1-8b-h100-cached/redis-ready.log | 342 - .../after-D1-8b-h100-cached/scheduler.log | 2 - .../after-D1-8b-h100-cached/timeline.json | 267 - .../after-D1-8b-h100-cached/watch-started.txt | 1 - .../after-D1-8b-h100-cached/worker-image.txt | 1 - .../after-D1-8b-h100-cached/workloads.json | 355 - .../after-D2-8b-h100-cached.run.log | 35 - .../after-D2-8b-h100-cached/api-server.log | 32 - .../after-D2-8b-h100-cached/client.jsonl | 11 - .../after-D2-8b-h100-cached/client.log | 12 - .../collect-started.txt | 1 - .../after-D2-8b-h100-cached/events.json | 3069 -------- .../log-orw-lora-qwen-qwen3-8b-0-sampler.txt | 222 - .../log-orw-lora-qwen-qwen3-8b-0-trainer.txt | 50 - .../node-images-after.json | 51 - .../node-images-before.json | 51 - .../node-labels-during.txt | 8 - .../after-D2-8b-h100-cached/pods.json | 6782 ---------------- .../after-D2-8b-h100-cached/redis-ready.log | 334 - .../after-D2-8b-h100-cached/scheduler.log | 2 - .../after-D2-8b-h100-cached/timeline.json | 267 - .../after-D2-8b-h100-cached/watch-started.txt | 1 - .../after-D2-8b-h100-cached/worker-image.txt | 1 - .../after-D2-8b-h100-cached/workloads.json | 355 - .../api-server.log | 26 - .../client.jsonl | 7 - .../client.log | 8 - .../collect-started.txt | 1 - .../events.json | 3265 -------- ...-orw-lora-google-gemma-4-e2b-0-trainer.txt | 32 - .../monitoring.json | 158 - .../node-images-after.json | 51 - .../node-images-before.json | 51 - .../pods.json | 6378 ---------------- .../redis-ready.log | 258 - .../scheduler.log | 1 - .../timeline.json | 130 - .../trainer-mountstats-final.txt | 3 - .../watch-started.txt | 1 - .../worker-image.txt | 1 - .../workloads.json | 179 - runs/2026-10-15_cold_start/after-progress.txt | 7 - runs/2026-10-15_cold_start/after.log | 0 runs/2026-10-15_cold_start/after.sh | 15 - .../fio-h100-1g3a-coldfile/file.txt | 1 - .../fio-randread-4k-direct.json | 260 - .../fio-seqread-1M-4jobs-direct.json | 261 - .../fio-seqread-1M-buffered.json | 260 - .../fio-seqread-1M-direct.json | 260 - .../fio-seqread-256k-qd1-direct.json | 260 - .../mountstats-after.txt | 31 - .../mountstats-before.txt | 32 - .../fio-l4-lvw8/file.txt | 1 - .../fio-l4-lvw8/fio-randread-4k-direct.json | 260 - .../fio-seqread-1M-4jobs-direct.json | 261 - .../fio-l4-lvw8/fio-seqread-1M-buffered.json | 260 - .../fio-l4-lvw8/fio-seqread-1M-direct.json | 260 - .../fio-seqread-256k-qd1-direct.json | 260 - .../fio-l4-lvw8/monitoring.json | 302 - .../fio-l4-lvw8/mountstats-after.txt | 31 - .../fio-l4-lvw8/mountstats-before.txt | 32 - runs/2026-10-15_cold_start/fix-image-tag.txt | 1 - .../node-labels-before.json | 42 - .../fleet-request-h200.md | 123 + runs/2026-10-17_demo_baselines/notes.md | 530 ++ 245 files changed, 653 insertions(+), 121937 deletions(-) delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/monitoring.json delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/monitoring.json delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/pods.json delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/timeline.json delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/B-0.5b-l4-cached/workloads.json delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold.run.log delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/api-server.log delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/client.log delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/events.json delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/monitoring.json delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached.run.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/client.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/events.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/monitoring.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/monitoring.json delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/node-labels-during.txt delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/monitoring.json delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/node-images-after.json delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/node-images-before.json delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/pods.json delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/redis-ready.log delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/scheduler.log delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/timeline.json delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/trainer-mountstats-final.txt delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/watch-started.txt delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/worker-image.txt delete mode 100644 runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/workloads.json delete mode 100644 runs/2026-10-15_cold_start/after-progress.txt delete mode 100644 runs/2026-10-15_cold_start/after.log delete mode 100755 runs/2026-10-15_cold_start/after.sh delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt delete mode 100644 runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt delete mode 100644 runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt delete mode 100644 runs/2026-10-15_cold_start/fix-image-tag.txt delete mode 100644 runs/2026-10-15_cold_start/node-labels-before.json create mode 100644 runs/2026-10-17_demo_baselines/fleet-request-h200.md create mode 100644 runs/2026-10-17_demo_baselines/notes.md diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log deleted file mode 100644 index f232d67d..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold.run.log +++ /dev/null @@ -1,34 +0,0 @@ -watching; recorders [95336, 95337, 95339, 95341, 95343, 95338, 95340, 95342] write into runs/2026-10-15_cold_start/A-0.5b-l4-cold -collected 2 worker log(s) into runs/2026-10-15_cold_start/A-0.5b-l4-cold -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.1 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 1.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | -300.0 | -| container create | trainer.event_Pulled | trainer.container_started | 473.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.4 | -| weights load | trainer.load_start | trainer.load_done | 4.3 | -| ready key | trainer.load_done | trainer.ready_key | -0.7 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 186.7 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.4 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.6 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 1.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | -470.0 | -| container create | sampler.event_Pulled | sampler.container_started | 642.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 11.9 | -| imports and engine setup until weights read | sampler.process_first_log | sampler.weights_loaded | 32.7 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.5 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.6 | -| ready key | sampler.engine_init_done | sampler.ready_key | 1.2 | -| client: first sample blocked | client.sample.start | client.sample.done | 277.0 | - -DONE=0 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log deleted file mode 100644 index d10c262a..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/api-server.log +++ /dev/null @@ -1,71 +0,0 @@ -2026-10-15T03:04:13.970+00:00 2026-10-15T03:04:13.965424776Z INFO: 10.32.3.1:49232 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:04:16.136+00:00 2026-10-15T03:04:16.130670078Z INFO: 127.0.0.1:55612 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T03:04:16.365+00:00 2026-10-15T03:04:16.360351230Z INFO: 127.0.0.1:55616 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T03:04:16.427+00:00 2026-10-15T03:04:16.423488415Z INFO: 127.0.0.1:55616 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T03:04:16.648+00:00 2026-10-15T03:04:16.643839900Z [API_SERVER] enqueued op=create_model request_id=ccc385bd-2c80-4be6-9ab7-0625542435ba model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T03:04:16.650+00:00 2026-10-15T03:04:16.644342790Z INFO: 127.0.0.1:55632 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T03:04:23.971+00:00 2026-10-15T03:04:23.964124117Z INFO: 10.32.3.1:36036 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:04:26.615+00:00 2026-10-15T03:04:26.609759344Z INFO: 127.0.0.1:48514 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:04:34.028+00:00 2026-10-15T03:04:33.964594425Z INFO: 10.32.3.1:33560 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:04:43.971+00:00 2026-10-15T03:04:43.964844670Z INFO: 10.32.3.1:40012 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:04:53.971+00:00 2026-10-15T03:04:53.964767187Z INFO: 10.32.3.1:40016 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:05:03.980+00:00 2026-10-15T03:05:03.965100240Z INFO: 10.32.3.1:34000 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:05:06.801+00:00 2026-10-15T03:05:06.795756734Z INFO: 127.0.0.1:42920 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:05:13.972+00:00 2026-10-15T03:05:13.964966273Z INFO: 10.32.3.1:55820 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:05:23.970+00:00 2026-10-15T03:05:23.965043231Z INFO: 10.32.3.1:48518 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:05:33.972+00:00 2026-10-15T03:05:33.965111192Z INFO: 10.32.3.1:56412 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:05:43.970+00:00 2026-10-15T03:05:43.964441881Z INFO: 10.32.3.1:40700 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:05:53.970+00:00 2026-10-15T03:05:53.964733835Z INFO: 10.32.3.1:59024 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:05:56.984+00:00 2026-10-15T03:05:56.978355679Z INFO: 127.0.0.1:38040 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:06:04.141+00:00 2026-10-15T03:06:03.965149122Z INFO: 10.32.3.1:54646 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:06:13.983+00:00 2026-10-15T03:06:13.964712249Z INFO: 10.32.3.1:48236 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:06:23.971+00:00 2026-10-15T03:06:23.964370498Z INFO: 10.32.3.1:44162 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:06:33.971+00:00 2026-10-15T03:06:33.965253254Z INFO: 10.32.3.1:49780 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:06:37.171+00:00 2026-10-15T03:06:37.165060181Z INFO: 127.0.0.1:34956 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:06:43.970+00:00 2026-10-15T03:06:43.964900145Z INFO: 10.32.3.1:44846 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:06:53.971+00:00 2026-10-15T03:06:53.964366759Z INFO: 10.32.3.1:58212 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:07:03.971+00:00 2026-10-15T03:07:03.964671419Z INFO: 10.32.3.1:59690 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:07:13.971+00:00 2026-10-15T03:07:13.964489989Z INFO: 10.32.3.1:46040 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:07:22.817+00:00 2026-10-15T03:07:22.810967004Z INFO: 127.0.0.1:59842 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T03:07:24.013+00:00 2026-10-15T03:07:23.965218186Z INFO: 10.32.3.1:47190 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:07:25.486+00:00 2026-10-15T03:07:25.480772722Z [API_SERVER] enqueued op=forward_backward request_id=c9b1b9ba-be87-47ed-8531-402cb2fabb4c model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T03:07:25.487+00:00 2026-10-15T03:07:25.481519493Z INFO: 127.0.0.1:59842 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T03:07:28.911+00:00 2026-10-15T03:07:28.906017913Z [API_SERVER] enqueued op=optim_step request_id=4eed00a3-56a3-46ac-9191-13fd5f7bcf0d model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T03:07:28.911+00:00 2026-10-15T03:07:28.906487435Z INFO: 127.0.0.1:59842 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T03:07:29.701+00:00 2026-10-15T03:07:29.697228415Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=0e4cfa12-b1fd-41e9-8346-b31a02c5d55f model_id=ccc385bd-2c80-4be6-9ab7-0625542435ba active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T03:07:29.703+00:00 2026-10-15T03:07:29.697821936Z INFO: 127.0.0.1:59842 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T03:07:30.370+00:00 2026-10-15T03:07:30.365760850Z INFO: 127.0.0.1:59860 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T03:07:33.971+00:00 2026-10-15T03:07:33.966052502Z INFO: 10.32.3.1:47144 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:07:43.972+00:00 2026-10-15T03:07:43.965551746Z INFO: 10.32.3.1:39724 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:07:53.974+00:00 2026-10-15T03:07:53.966118583Z INFO: 10.32.3.1:52036 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:08:04.003+00:00 2026-10-15T03:08:03.965792751Z INFO: 10.32.3.1:53772 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:08:13.972+00:00 2026-10-15T03:08:13.965952960Z INFO: 10.32.3.1:55912 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:08:17.378+00:00 2026-10-15T03:08:17.363707043Z INFO: 127.0.0.1:40350 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:08:23.972+00:00 2026-10-15T03:08:23.965275434Z INFO: 10.32.3.1:37596 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:08:34.056+00:00 2026-10-15T03:08:33.964948070Z INFO: 10.32.3.1:55858 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:08:43.973+00:00 2026-10-15T03:08:43.966182718Z INFO: 10.32.3.1:33098 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:08:53.973+00:00 2026-10-15T03:08:53.965614609Z INFO: 10.32.3.1:38942 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:09:03.972+00:00 2026-10-15T03:09:03.964678901Z INFO: 10.32.3.1:45794 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:09:07.564+00:00 2026-10-15T03:09:07.556603346Z INFO: 127.0.0.1:47276 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:09:13.976+00:00 2026-10-15T03:09:13.967273256Z INFO: 10.32.3.1:54252 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:09:16.493+00:00 2026-10-15T03:09:16.484867631Z INFO: 127.0.0.1:57588 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T03:09:23.972+00:00 2026-10-15T03:09:23.964796053Z INFO: 10.32.3.1:60326 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:09:33.973+00:00 2026-10-15T03:09:33.965636084Z INFO: 10.32.3.1:58520 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:09:43.973+00:00 2026-10-15T03:09:43.965716052Z INFO: 10.32.3.1:46004 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:09:53.976+00:00 2026-10-15T03:09:53.966124758Z INFO: 10.32.3.1:58072 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:09:57.751+00:00 2026-10-15T03:09:57.744852851Z INFO: 127.0.0.1:60840 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:10:03.975+00:00 2026-10-15T03:10:03.966670840Z INFO: 10.32.3.1:58060 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:10:13.975+00:00 2026-10-15T03:10:13.966758337Z INFO: 10.32.3.1:57718 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:10:23.974+00:00 2026-10-15T03:10:23.964882780Z INFO: 10.32.3.1:33186 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:10:33.975+00:00 2026-10-15T03:10:33.966514608Z INFO: 10.32.3.1:45636 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:10:37.927+00:00 2026-10-15T03:10:37.918752812Z INFO: 127.0.0.1:33472 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:10:43.984+00:00 2026-10-15T03:10:43.965936582Z INFO: 10.32.3.1:50548 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:10:53.974+00:00 2026-10-15T03:10:53.965049215Z INFO: 10.32.3.1:56646 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:11:03.975+00:00 2026-10-15T03:11:03.965725149Z INFO: 10.32.3.1:46908 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:11:13.971+00:00 2026-10-15T03:11:13.964776008Z INFO: 10.32.3.1:59014 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:11:23.973+00:00 2026-10-15T03:11:23.964881664Z INFO: 10.32.3.1:37578 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:11:33.972+00:00 2026-10-15T03:11:33.964840171Z INFO: 10.32.3.1:57742 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:11:38.106+00:00 2026-10-15T03:11:38.100543048Z INFO: 127.0.0.1:49588 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:11:43.973+00:00 2026-10-15T03:11:43.965228717Z INFO: 10.32.3.1:60810 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:11:53.972+00:00 2026-10-15T03:11:53.964561540Z INFO: 10.32.3.1:56102 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:12:03.973+00:00 2026-10-15T03:12:03.965618560Z INFO: 10.32.3.1:33308 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl deleted file mode 100644 index 2ab7d41f..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T03:04:15.891+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T03:04:15.892+00:00", "t": 0.0, "phase": "create_model.start"} -{"ts": "2026-10-15T03:07:22.617+00:00", "t": 186.725, "phase": "create_model.done", "model_id": "ccc385bd-2c80-4be6-9ab7-0625542435ba"} -{"ts": "2026-10-15T03:07:25.417+00:00", "t": 189.525, "phase": "forward_backward.start"} -{"ts": "2026-10-15T03:07:28.837+00:00", "t": 192.945, "phase": "forward_backward.done"} -{"ts": "2026-10-15T03:07:29.600+00:00", "t": 193.709, "phase": "optim_step.done"} -{"ts": "2026-10-15T03:07:29.600+00:00", "t": 193.709, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T03:07:30.173+00:00", "t": 194.282, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T03:07:30.174+00:00", "t": 194.282, "phase": "sample.start"} -{"ts": "2026-10-15T03:12:07.151+00:00", "t": 471.26, "phase": "sample.done", "text": " 4"} -{"ts": "2026-10-15T03:12:07.154+00:00", "t": 471.262, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log deleted file mode 100644 index 0f6171ae..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.000s create_model.start -[cold-start] 186.725s create_model.done {'model_id': 'ccc385bd-2c80-4be6-9ab7-0625542435ba'} -[cold-start] 189.525s forward_backward.start -[cold-start] 192.945s forward_backward.done -[cold-start] 193.709s optim_step.done -[cold-start] 193.709s save_weights_and_get_sampling_client.start -[cold-start] 194.282s save_weights_and_get_sampling_client.done -[cold-start] 194.282s sample.start -[cold-start] 471.260s sample.done {'text': ' 4'} -[cold-start] 471.262s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt deleted file mode 100644 index 45604b3b..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T03:12:07.681+00:00 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json deleted file mode 100644 index 74c9ca47..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/events.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:36Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176697983001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:36Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:36Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94ae008553cd", - "namespace": "openrl-system", - "resourceVersion": "1792033176815007016", - "uid": "363bd5d3-147c-4b0d-8500-d3a3b6232789" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:40Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176788543001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:40Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:40Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed139d233", - "namespace": "openrl-system", - "resourceVersion": "1792033180304687014", - "uid": "2463b9b8-32de-4c33-af86-ca945ab359e3" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:40Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176788543001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:40Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:40Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed2f06544", - "namespace": "openrl-system", - "resourceVersion": "1792033180345279015", - "uid": "4766979d-5fb4-4934-96f7-b8e5c1833efa" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:40Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176788543001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:40Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:40Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aedddd6086", - "namespace": "openrl-system", - "resourceVersion": "1792033180517807014", - "uid": "e1c7e13f-7b29-4ed9-afdb-cc1a062a32d2" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:03:24Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176788543001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:03:24Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:03:24Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94e2ff112e95", - "namespace": "openrl-system", - "resourceVersion": "1792033404430399002", - "uid": "2f914e49-5325-4325-b2af-29bca754d7d2" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:07:30Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033649966431001", - "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:07:30Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T03:07:30Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de951c3308c78d", - "namespace": "openrl-system", - "resourceVersion": "1792033650098607013", - "uid": "703ebdf3-49c5-40b9-abd8-9ae5be2441e0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:07:33Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033650081775001", - "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:07:33Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", - "metadata": { - "creationTimestamp": "2026-10-15T03:07:33Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de951d16e18ede", - "namespace": "openrl-system", - "resourceVersion": "1792033653954879017", - "uid": "b6907b80-ca84-45a3-bf35-1cc63a9f9e88" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:10:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033650081775001", - "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:10:22Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.566s (2m48.566s including waiting). Image size: 7123551739 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T03:10:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de954456385799", - "namespace": "openrl-system", - "resourceVersion": "1792033822509983001", - "uid": "14bc5358-d229-48a6-aea1-586d2b488ba2" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:10:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033650081775001", - "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:10:22Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T03:10:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de95445d54c744", - "namespace": "openrl-system", - "resourceVersion": "1792033822614143018", - "uid": "c54769e6-3c1e-4740-9a25-03c3b3f2f2c8" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:10:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033650081775001", - "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:10:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T03:10:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de954472a82f0e", - "namespace": "openrl-system", - "resourceVersion": "1792033822991583015", - "uid": "3e74963a-d669-4b13-8870-db29be3feee1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153094671012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a883fadef0", - "namespace": "openrl-system", - "resourceVersion": "1792033153265231011", - "uid": "62c5cae6-2deb-43a0-a097-73157e760539" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:17Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153224047012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:17Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:17Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a9671f24f0", - "namespace": "openrl-system", - "resourceVersion": "1792033157084143008", - "uid": "9472795b-31b9-478f-a545-6146795d5503" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:17Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153224047012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:17Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:17Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a968b7d635", - "namespace": "openrl-system", - "resourceVersion": "1792033157103615024", - "uid": "11d1cf36-8464-42c3-94e2-1843723223c8" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:17Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153224047012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:17Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:17Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a973285d70", - "namespace": "openrl-system", - "resourceVersion": "1792033157264271007", - "uid": "8fa654a4-608f-4e8c-804f-9d1d5c91650a" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:03:23Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153224047012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:03:23Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:03:23Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94e2d9f9f25a", - "namespace": "openrl-system", - "resourceVersion": "1792033403812895007", - "uid": "acb91806-a2a8-401c-af8a-60b6035618d4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:04:17Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033457005855012", - "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:04:17Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T03:04:17Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94ef537aea10", - "namespace": "openrl-system", - "resourceVersion": "1792033457378303014", - "uid": "c2b1e3e0-7483-4202-a802-114b81a3f82b" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:04:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033457353199012", - "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:04:21Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", - "metadata": { - "creationTimestamp": "2026-10-15T03:04:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94f037d8b52d", - "namespace": "openrl-system", - "resourceVersion": "1792033461208399017", - "uid": "b6d87d48-8207-4dc8-95cb-72d018a65a89" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:07:09Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033457353199012", - "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:07:09Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.59s (2m48.59s including waiting). Image size: 7123551739 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T03:07:09Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de951778a75e12", - "namespace": "openrl-system", - "resourceVersion": "1792033629797471021", - "uid": "9f72c5d6-3d2e-40cc-97a1-d829da2bdcf6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:07:09Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033457353199012", - "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:07:09Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T03:07:09Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de95177ce47bff", - "namespace": "openrl-system", - "resourceVersion": "1792033629877183016", - "uid": "cff96884-931d-4efe-9b36-7e38e39d1ab6" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:07:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033457353199012", - "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:07:10Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T03:07:10Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de95178de8c928", - "namespace": "openrl-system", - "resourceVersion": "1792033630159423023", - "uid": "99e1e2c0-7347-45f8-ae1d-2c56266651d1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt deleted file mode 100644 index 9affdaaf..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt +++ /dev/null @@ -1,120 +0,0 @@ -2026-10-15T03:10:33.926555905Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T03:10:46.387987219Z INFO 10-15 03:10:46 [model.py:692] Resolved architecture: Qwen2ForCausalLM -2026-10-15T03:10:46.388270451Z INFO 10-15 03:10:46 [model.py:2030] Using max model len 8192 -2026-10-15T03:10:46.390489193Z INFO 10-15 03:10:46 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. -2026-10-15T03:10:46.761209045Z INFO 10-15 03:10:46 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T03:10:50.185230245Z WARNING 10-15 03:10:50 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T03:11:01.668050858Z (EngineCore pid=161) INFO 10-15 03:11:01 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T03:11:02.480729209Z (EngineCore pid=161) INFO 10-15 03:11:02 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_af67dc74611246bb99d63df9f6327803 backend=nccl -2026-10-15T03:11:02.487863050Z (EngineCore pid=161) INFO 10-15 03:11:02 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T03:11:02.487892790Z (EngineCore pid=161) INFO 10-15 03:11:02 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T03:11:03.371362030Z (EngineCore pid=161) INFO 10-15 03:11:03 [model_runner.py:396] Loading model from scratch... -2026-10-15T03:11:04.235459935Z (EngineCore pid=161) INFO 10-15 03:11:04 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T03:11:04.235903689Z (EngineCore pid=161) INFO 10-15 03:11:04 [flash_attn.py:1116] Using FlashAttention version 2 -2026-10-15T03:11:04.354712405Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T03:11:04.355179834Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T03:11:04.731296224Z (EngineCore pid=161) INFO 10-15 03:11:04 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.62 GiB. -2026-10-15T03:11:04.731338778Z (EngineCore pid=161) INFO 10-15 03:11:04 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T03:11:04.732630306Z (EngineCore pid=161) -Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T03:07:17.275476076Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T03:07:17.307183476Z -2026-10-15T03:07:17.410020160Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... -2026-10-15T03:07:17.543492303Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T03:07:21.568855096Z -Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 24h v1.35.8-gke.1380001 false -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 24h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json deleted file mode 100644 index 71e4ba3c..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-api-server-645ff9777d-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "645ff9777d", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - } - ], - "resourceVersion": "1791941173996783021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-79xdl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:02Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:26:02Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.10", - "podIPs": [ - { - "ip": "10.32.3.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T03:07:29Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "uid": "54443ec4-9700-416e-b5d7-846a4a007c00" - } - ], - "resourceVersion": "1792033823293583001", - "uid": "6de07128-ec40-4eaa-924e-3512b49c90cf" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "d96cccda-acfd-4fa4-95c5-72c958a83f6b" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zfr8c", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-zfr8c", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:10:23Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:07:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:10:23Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:10:23Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:07:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "25Gi" - }, - "containerID": "containerd://2183700bb5c7f07cd82180b35a53253d88b404e2508623e204e56ec7fc76674b", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T03:10:22Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zfr8c", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.0.19", - "podIPs": [ - { - "ip": "10.32.0.19" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T03:07:30Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T03:04:16Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "uid": "47b54db3-9dc6-4dfd-a972-55bee1fc0dad" - } - ], - "resourceVersion": "1792033631097647012", - "uid": "041cef29-6ab7-498d-be07-e038fc82b40d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "33cc03f0-42b9-4860-8d5e-f14ca74f3511" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2b24h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2b24h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:07:11Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:04:17Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:07:11Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:07:11Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:04:17Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "21Gi" - }, - "containerID": "containerd://1e1957d4c7ed76e3bcb86d4334323d28c34b994664a0e8e29b4f551d8f8f5b99", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T03:07:10Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2b24h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.13.24", - "podIPs": [ - { - "ip": "10.32.13.24" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T03:04:17Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log deleted file mode 100644 index a3b2f618..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/redis-ready.log +++ /dev/null @@ -1,299 +0,0 @@ -2026-10-15T03:07:21.981+00:00 2026-10-15T03:07:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:22.984+00:00 2026-10-15T03:07:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:24.013+00:00 2026-10-15T03:07:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:24.993+00:00 2026-10-15T03:07:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:25.999+00:00 2026-10-15T03:07:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:27.005+00:00 2026-10-15T03:07:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:28.011+00:00 2026-10-15T03:07:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:29.016+00:00 2026-10-15T03:07:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:30.021+00:00 2026-10-15T03:07:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:31.026+00:00 2026-10-15T03:07:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:32.031+00:00 2026-10-15T03:07:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:33.036+00:00 2026-10-15T03:07:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:34.041+00:00 2026-10-15T03:07:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:35.046+00:00 2026-10-15T03:07:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:36.051+00:00 2026-10-15T03:07:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:37.056+00:00 2026-10-15T03:07:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:38.062+00:00 2026-10-15T03:07:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:39.068+00:00 2026-10-15T03:07:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:40.072+00:00 2026-10-15T03:07:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:41.077+00:00 2026-10-15T03:07:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:42.083+00:00 2026-10-15T03:07:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:43.088+00:00 2026-10-15T03:07:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:44.098+00:00 2026-10-15T03:07:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:45.098+00:00 2026-10-15T03:07:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:46.105+00:00 2026-10-15T03:07:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:47.109+00:00 2026-10-15T03:07:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:48.116+00:00 2026-10-15T03:07:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:49.118+00:00 2026-10-15T03:07:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:50.124+00:00 2026-10-15T03:07:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:51.129+00:00 2026-10-15T03:07:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:52.139+00:00 2026-10-15T03:07:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:53.140+00:00 2026-10-15T03:07:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:54.145+00:00 2026-10-15T03:07:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:55.150+00:00 2026-10-15T03:07:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:56.154+00:00 2026-10-15T03:07:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:57.163+00:00 2026-10-15T03:07:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:58.164+00:00 2026-10-15T03:07:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:07:59.169+00:00 2026-10-15T03:07:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:00.174+00:00 2026-10-15T03:08:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:01.180+00:00 2026-10-15T03:08:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:02.184+00:00 2026-10-15T03:08:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:03.190+00:00 2026-10-15T03:08:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:04.194+00:00 2026-10-15T03:08:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:05.213+00:00 2026-10-15T03:08:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:06.413+00:00 2026-10-15T03:08:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:07.208+00:00 2026-10-15T03:08:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:08.214+00:00 2026-10-15T03:08:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:09.219+00:00 2026-10-15T03:08:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:10.235+00:00 2026-10-15T03:08:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:11.229+00:00 2026-10-15T03:08:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:12.234+00:00 2026-10-15T03:08:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:13.248+00:00 2026-10-15T03:08:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:14.245+00:00 2026-10-15T03:08:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:15.252+00:00 2026-10-15T03:08:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:16.257+00:00 2026-10-15T03:08:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:17.262+00:00 2026-10-15T03:08:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:18.269+00:00 2026-10-15T03:08:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:19.272+00:00 2026-10-15T03:08:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:20.284+00:00 2026-10-15T03:08:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:21.282+00:00 2026-10-15T03:08:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:22.289+00:00 2026-10-15T03:08:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:23.292+00:00 2026-10-15T03:08:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:24.298+00:00 2026-10-15T03:08:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:25.455+00:00 2026-10-15T03:08:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:26.309+00:00 2026-10-15T03:08:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:27.313+00:00 2026-10-15T03:08:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:28.320+00:00 2026-10-15T03:08:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:29.330+00:00 2026-10-15T03:08:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:30.328+00:00 2026-10-15T03:08:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:31.333+00:00 2026-10-15T03:08:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:32.339+00:00 2026-10-15T03:08:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:33.342+00:00 2026-10-15T03:08:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:34.352+00:00 2026-10-15T03:08:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:35.362+00:00 2026-10-15T03:08:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:36.358+00:00 2026-10-15T03:08:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:37.362+00:00 2026-10-15T03:08:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:38.367+00:00 2026-10-15T03:08:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:39.374+00:00 2026-10-15T03:08:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:40.382+00:00 2026-10-15T03:08:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:41.382+00:00 2026-10-15T03:08:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:42.388+00:00 2026-10-15T03:08:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:43.392+00:00 2026-10-15T03:08:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:44.398+00:00 2026-10-15T03:08:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:45.403+00:00 2026-10-15T03:08:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:46.408+00:00 2026-10-15T03:08:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:47.417+00:00 2026-10-15T03:08:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:48.419+00:00 2026-10-15T03:08:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:49.452+00:00 2026-10-15T03:08:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:50.428+00:00 2026-10-15T03:08:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:51.436+00:00 2026-10-15T03:08:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:52.440+00:00 2026-10-15T03:08:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:53.445+00:00 2026-10-15T03:08:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:54.451+00:00 2026-10-15T03:08:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:55.456+00:00 2026-10-15T03:08:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:56.465+00:00 2026-10-15T03:08:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:57.465+00:00 2026-10-15T03:08:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:58.470+00:00 2026-10-15T03:08:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:08:59.476+00:00 2026-10-15T03:08:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:00.480+00:00 2026-10-15T03:09:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:01.484+00:00 2026-10-15T03:09:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:02.490+00:00 2026-10-15T03:09:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:03.495+00:00 2026-10-15T03:09:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:04.501+00:00 2026-10-15T03:09:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:05.511+00:00 2026-10-15T03:09:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:06.510+00:00 2026-10-15T03:09:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:07.516+00:00 2026-10-15T03:09:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:08.653+00:00 2026-10-15T03:09:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:09.524+00:00 2026-10-15T03:09:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:10.667+00:00 2026-10-15T03:09:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:11.537+00:00 2026-10-15T03:09:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:12.540+00:00 2026-10-15T03:09:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:13.546+00:00 2026-10-15T03:09:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:14.556+00:00 2026-10-15T03:09:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:15.556+00:00 2026-10-15T03:09:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:16.562+00:00 2026-10-15T03:09:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:17.565+00:00 2026-10-15T03:09:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:18.704+00:00 2026-10-15T03:09:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:19.576+00:00 2026-10-15T03:09:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:20.581+00:00 2026-10-15T03:09:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:21.589+00:00 2026-10-15T03:09:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:22.637+00:00 2026-10-15T03:09:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:23.599+00:00 2026-10-15T03:09:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:24.603+00:00 2026-10-15T03:09:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:25.608+00:00 2026-10-15T03:09:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:26.613+00:00 2026-10-15T03:09:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:27.629+00:00 2026-10-15T03:09:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:28.628+00:00 2026-10-15T03:09:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:29.628+00:00 2026-10-15T03:09:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:30.633+00:00 2026-10-15T03:09:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:31.638+00:00 2026-10-15T03:09:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:32.642+00:00 2026-10-15T03:09:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:33.648+00:00 2026-10-15T03:09:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:34.653+00:00 2026-10-15T03:09:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:35.659+00:00 2026-10-15T03:09:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:36.728+00:00 2026-10-15T03:09:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:37.669+00:00 2026-10-15T03:09:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:38.674+00:00 2026-10-15T03:09:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:39.679+00:00 2026-10-15T03:09:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:40.686+00:00 2026-10-15T03:09:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:41.850+00:00 2026-10-15T03:09:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:42.695+00:00 2026-10-15T03:09:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:43.701+00:00 2026-10-15T03:09:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:44.706+00:00 2026-10-15T03:09:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:45.710+00:00 2026-10-15T03:09:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:46.719+00:00 2026-10-15T03:09:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:47.721+00:00 2026-10-15T03:09:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:48.726+00:00 2026-10-15T03:09:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:49.731+00:00 2026-10-15T03:09:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:50.736+00:00 2026-10-15T03:09:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:51.743+00:00 2026-10-15T03:09:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:52.747+00:00 2026-10-15T03:09:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:53.753+00:00 2026-10-15T03:09:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:54.757+00:00 2026-10-15T03:09:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:55.776+00:00 2026-10-15T03:09:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:56.771+00:00 2026-10-15T03:09:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:57.772+00:00 2026-10-15T03:09:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:58.778+00:00 2026-10-15T03:09:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:09:59.783+00:00 2026-10-15T03:09:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:00.787+00:00 2026-10-15T03:10:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:01.793+00:00 2026-10-15T03:10:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:02.799+00:00 2026-10-15T03:10:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:03.804+00:00 2026-10-15T03:10:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:04.809+00:00 2026-10-15T03:10:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:05.815+00:00 2026-10-15T03:10:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:06.819+00:00 2026-10-15T03:10:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:07.828+00:00 2026-10-15T03:10:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:08.829+00:00 2026-10-15T03:10:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:09.835+00:00 2026-10-15T03:10:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:10.845+00:00 2026-10-15T03:10:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:11.850+00:00 2026-10-15T03:10:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:12.851+00:00 2026-10-15T03:10:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:13.856+00:00 2026-10-15T03:10:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:14.862+00:00 2026-10-15T03:10:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:15.868+00:00 2026-10-15T03:10:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:16.872+00:00 2026-10-15T03:10:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:17.878+00:00 2026-10-15T03:10:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:18.884+00:00 2026-10-15T03:10:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:19.888+00:00 2026-10-15T03:10:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:20.982+00:00 2026-10-15T03:10:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:21.899+00:00 2026-10-15T03:10:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:22.904+00:00 2026-10-15T03:10:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:23.910+00:00 2026-10-15T03:10:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:24.914+00:00 2026-10-15T03:10:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:25.919+00:00 2026-10-15T03:10:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:26.934+00:00 2026-10-15T03:10:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:27.929+00:00 2026-10-15T03:10:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:28.937+00:00 2026-10-15T03:10:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:29.940+00:00 2026-10-15T03:10:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:30.946+00:00 2026-10-15T03:10:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:31.949+00:00 2026-10-15T03:10:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:32.955+00:00 2026-10-15T03:10:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:33.960+00:00 2026-10-15T03:10:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:34.966+00:00 2026-10-15T03:10:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:35.972+00:00 2026-10-15T03:10:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:36.976+00:00 2026-10-15T03:10:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:37.982+00:00 2026-10-15T03:10:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:38.987+00:00 2026-10-15T03:10:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:40.004+00:00 2026-10-15T03:10:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:40.999+00:00 2026-10-15T03:10:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:42.003+00:00 2026-10-15T03:10:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:43.009+00:00 2026-10-15T03:10:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:44.015+00:00 2026-10-15T03:10:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:45.029+00:00 2026-10-15T03:10:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:46.044+00:00 2026-10-15T03:10:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:47.030+00:00 2026-10-15T03:10:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:48.037+00:00 2026-10-15T03:10:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:49.040+00:00 2026-10-15T03:10:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:50.048+00:00 2026-10-15T03:10:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:51.050+00:00 2026-10-15T03:10:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:52.055+00:00 2026-10-15T03:10:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:53.063+00:00 2026-10-15T03:10:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:54.065+00:00 2026-10-15T03:10:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:55.071+00:00 2026-10-15T03:10:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:56.076+00:00 2026-10-15T03:10:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:57.081+00:00 2026-10-15T03:10:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:58.086+00:00 2026-10-15T03:10:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:10:59.091+00:00 2026-10-15T03:10:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:00.096+00:00 2026-10-15T03:11:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:01.109+00:00 2026-10-15T03:11:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:02.106+00:00 2026-10-15T03:11:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:03.111+00:00 2026-10-15T03:11:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:04.118+00:00 2026-10-15T03:11:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:05.123+00:00 2026-10-15T03:11:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:06.131+00:00 2026-10-15T03:11:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:07.138+00:00 2026-10-15T03:11:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:08.141+00:00 2026-10-15T03:11:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:09.180+00:00 2026-10-15T03:11:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:10.150+00:00 2026-10-15T03:11:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:11.213+00:00 2026-10-15T03:11:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:12.162+00:00 2026-10-15T03:11:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:13.164+00:00 2026-10-15T03:11:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:14.169+00:00 2026-10-15T03:11:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:15.175+00:00 2026-10-15T03:11:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:16.187+00:00 2026-10-15T03:11:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:17.185+00:00 2026-10-15T03:11:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:18.190+00:00 2026-10-15T03:11:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:19.196+00:00 2026-10-15T03:11:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:20.201+00:00 2026-10-15T03:11:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:21.206+00:00 2026-10-15T03:11:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:22.211+00:00 2026-10-15T03:11:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:23.217+00:00 2026-10-15T03:11:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:24.221+00:00 2026-10-15T03:11:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:25.226+00:00 2026-10-15T03:11:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:26.230+00:00 2026-10-15T03:11:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:27.236+00:00 2026-10-15T03:11:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:28.247+00:00 2026-10-15T03:11:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:29.246+00:00 2026-10-15T03:11:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:30.252+00:00 2026-10-15T03:11:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:31.256+00:00 2026-10-15T03:11:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:32.273+00:00 2026-10-15T03:11:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:33.272+00:00 2026-10-15T03:11:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:34.272+00:00 2026-10-15T03:11:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:35.277+00:00 2026-10-15T03:11:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:36.282+00:00 2026-10-15T03:11:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:37.298+00:00 2026-10-15T03:11:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:38.292+00:00 2026-10-15T03:11:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:39.298+00:00 2026-10-15T03:11:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:40.303+00:00 2026-10-15T03:11:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:41.308+00:00 2026-10-15T03:11:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:42.313+00:00 2026-10-15T03:11:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:43.318+00:00 2026-10-15T03:11:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:44.323+00:00 2026-10-15T03:11:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:45.329+00:00 2026-10-15T03:11:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:46.336+00:00 2026-10-15T03:11:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:47.357+00:00 2026-10-15T03:11:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:48.344+00:00 2026-10-15T03:11:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:49.386+00:00 2026-10-15T03:11:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:50.353+00:00 2026-10-15T03:11:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:51.358+00:00 2026-10-15T03:11:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:52.363+00:00 2026-10-15T03:11:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:53.376+00:00 2026-10-15T03:11:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:54.374+00:00 2026-10-15T03:11:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:55.378+00:00 2026-10-15T03:11:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:56.385+00:00 2026-10-15T03:11:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:57.389+00:00 2026-10-15T03:11:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:58.393+00:00 2026-10-15T03:11:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:11:59.398+00:00 2026-10-15T03:11:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:00.403+00:00 2026-10-15T03:12:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:01.491+00:00 2026-10-15T03:12:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:02.414+00:00 2026-10-15T03:12:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:03.417+00:00 2026-10-15T03:12:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:04.423+00:00 2026-10-15T03:12:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:05.427+00:00 2026-10-15T03:12:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:06.434+00:00 2026-10-15T03:12:06Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:06.434+00:00 2026-10-15T03:12:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:07.438+00:00 2026-10-15T03:12:07Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:07.438+00:00 2026-10-15T03:12:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:08.445+00:00 2026-10-15T03:12:08Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:08.445+00:00 2026-10-15T03:12:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:09.456+00:00 2026-10-15T03:12:09Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:09.456+00:00 2026-10-15T03:12:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:10.454+00:00 2026-10-15T03:12:10Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:10.454+00:00 2026-10-15T03:12:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:11.462+00:00 2026-10-15T03:12:11Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:11.462+00:00 2026-10-15T03:12:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:12.466+00:00 2026-10-15T03:12:12Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:12.466+00:00 2026-10-15T03:12:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:13.470+00:00 2026-10-15T03:12:13Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:12:13.470+00:00 2026-10-15T03:12:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log deleted file mode 100644 index 8a5d22c7..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T03:04:16.865+00:00 2026-10-15T03:04:16.851951680Z {"level":"info","ts":"2026-10-15T03:04:16Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"4c2fe683-2517-4a3a-b018-18a9782b7839","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} -2026-10-15T03:07:29.839+00:00 2026-10-15T03:07:29.833404182Z {"level":"info","ts":"2026-10-15T03:07:29Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"ed0ac0cd-45a6-4012-adaa-1fd004dac781","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json deleted file mode 100644 index a61a4e27..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/timeline.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "case": "A-0.5b-l4-cold", - "collected": "2026-10-15T03:38:49.515+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 03:04:15.891000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen2.5-0.5B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 03:04:15.892000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 03:07:22.617000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "ccc385bd-2c80-4be6-9ab7-0625542435ba" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 03:07:25.417000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 03:07:28.837000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 03:07:29.600000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 03:07:29.600000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 03:07:30.173000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 03:07:30.174000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 03:12:07.151000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4" - }, - "client.end": { - "ts": "2026-10-15 03:12:07.154000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 03:07:29+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 03:07:28.976000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 03:07:30.816000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 03:07:30.816000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 03:10:23+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 03:07:29+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 03:07:30+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 03:10:23+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 03:10:23+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 03:07:30+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 03:10:22+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 03:07:30+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "sampler.event_Pulling": { - "ts": "2026-10-15 03:07:33+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 03:10:22+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.566s (2m48.566s i", - "kubelet_pull_s": 168.566, - "pulled_from_cache": false - }, - "sampler.event_Created": { - "ts": "2026-10-15 03:10:22+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 03:10:22+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 03:10:33.926555+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 03:11:01.668050+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:01 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 03:11:03.371362+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:03 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 03:11:04.731338+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:04 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 03:11:06.644316+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:06 [default_loader.py:430] Loading weights took 1.92 seconds", - "self_reported_s": 1.92 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 03:11:07.445238+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:07 [model_runner.py:428] Model loading took 1.47 GiB memory and 4.387871 seconds", - "self_reported_s": 4.387871 - }, - "sampler.compile_done": { - "ts": "2026-10-15 03:11:38.154127+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:38 [monitor.py:53] torch.compile took 28.48 s in total", - "self_reported_s": 28.48 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 03:11:53.539804+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:53 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", - "self_reported_s": 11.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 03:11:54.078717+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:11:54 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 03:12:04.781237+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 03:12:04 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.33 s", - "self_reported_s": 57.33 - }, - "sampler.ready_key": { - "ts": "2026-10-15 03:12:06+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" - }, - "trainer": { - "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 03:04:16+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 03:04:16.578000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 03:04:16.578000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 03:04:18.224000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 03:07:11+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 03:04:16+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 03:04:17+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 03:07:11+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 03:07:11+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 03:04:17+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 03:07:10+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 03:04:17+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "trainer.event_Pulling": { - "ts": "2026-10-15 03:04:21+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 03:07:09+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 2m48.59s (2m48.59s inc", - "kubelet_pull_s": 168.59, - "pulled_from_cache": false - }, - "trainer.event_Created": { - "ts": "2026-10-15 03:07:09+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 03:07:10+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 03:07:17.253128+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 03:07:17.410020+00:00", - "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 03:07:21.686686+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 03:07:21.689422+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 03:07:21+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt deleted file mode 100644 index 3d7afe64..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T03:04:11.775+00:00 diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt deleted file mode 100644 index 0d162c07..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json b/runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json deleted file mode 100644 index 28c8d645..00000000 --- a/runs/2026-10-15_cold_start/A-0.5b-l4-cold/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T03:07:29Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033823440495001", - "uid": "54443ec4-9700-416e-b5d7-846a4a007c00" - }, - "spec": { - "accelerator": { - "memory": "4Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "name": "worker", - "resources": { - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "d96cccda-acfd-4fa4-95c5-72c958a83f6b", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T03:10:23Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-54443ec4", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "4Gi", - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T03:04:16Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033631210927015", - "uid": "47b54db3-9dc6-4dfd-a972-55bee1fc0dad" - }, - "spec": { - "accelerator": { - "memory": "5Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "name": "worker", - "resources": { - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "33cc03f0-42b9-4860-8d5e-f14ca74f3511", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T03:07:11Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-47b54db3", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "5Gi", - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log deleted file mode 100644 index e3ebe340..00000000 --- a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/api-server.log +++ /dev/null @@ -1,31 +0,0 @@ -2026-10-15T02:59:03.962+00:00 2026-10-15T02:59:03.964951462Z INFO: 10.32.3.1:54528 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T02:59:11.901+00:00 2026-10-15T02:59:11.904243760Z INFO: 127.0.0.1:55772 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T02:59:12.147+00:00 2026-10-15T02:59:12.151769067Z INFO: 127.0.0.1:55782 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T02:59:12.211+00:00 2026-10-15T02:59:12.215619037Z INFO: 127.0.0.1:55782 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T02:59:12.479+00:00 2026-10-15T02:59:12.482819930Z [API_SERVER] enqueued op=create_model request_id=cc688d59-8147-4315-9236-81ea9296fa1d model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T02:59:12.481+00:00 2026-10-15T02:59:12.483378195Z INFO: 127.0.0.1:55792 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T02:59:13.961+00:00 2026-10-15T02:59:13.965191360Z INFO: 10.32.3.1:48124 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T02:59:22.388+00:00 2026-10-15T02:59:22.390801550Z INFO: 127.0.0.1:45904 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T02:59:23.961+00:00 2026-10-15T02:59:23.964632011Z INFO: 10.32.3.1:39380 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T02:59:29.361+00:00 2026-10-15T02:59:29.363125454Z INFO: 127.0.0.1:45914 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T02:59:32.441+00:00 2026-10-15T02:59:32.444410717Z [API_SERVER] enqueued op=forward_backward request_id=0aefaadb-3d41-4420-89c5-bf8de12892b6 model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T02:59:32.441+00:00 2026-10-15T02:59:32.445344260Z INFO: 127.0.0.1:45914 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T02:59:33.961+00:00 2026-10-15T02:59:33.964713101Z INFO: 10.32.3.1:51494 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T02:59:35.560+00:00 2026-10-15T02:59:35.564042664Z [API_SERVER] enqueued op=optim_step request_id=cf94adcb-6e7f-47ac-97bc-f5de0168c89b model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T02:59:35.561+00:00 2026-10-15T02:59:35.564697067Z INFO: 127.0.0.1:45914 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T02:59:36.462+00:00 2026-10-15T02:59:36.445520836Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=d5021fb2-234b-470a-b272-1ad3d13c1ef6 model_id=cc688d59-8147-4315-9236-81ea9296fa1d active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T02:59:36.462+00:00 2026-10-15T02:59:36.446145534Z INFO: 127.0.0.1:45914 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T02:59:37.136+00:00 2026-10-15T02:59:37.140404264Z INFO: 127.0.0.1:59990 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T02:59:43.962+00:00 2026-10-15T02:59:43.964864449Z INFO: 10.32.3.1:53026 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T02:59:53.962+00:00 2026-10-15T02:59:53.965184980Z INFO: 10.32.3.1:58910 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:00:03.962+00:00 2026-10-15T03:00:03.964285482Z INFO: 10.32.3.1:44748 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:00:13.962+00:00 2026-10-15T03:00:13.964568310Z INFO: 10.32.3.1:55124 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:00:22.589+00:00 2026-10-15T03:00:22.590580301Z INFO: 127.0.0.1:41576 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:00:23.962+00:00 2026-10-15T03:00:23.965334932Z INFO: 10.32.3.1:37274 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:00:33.962+00:00 2026-10-15T03:00:33.965008993Z INFO: 10.32.3.1:48328 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:00:43.962+00:00 2026-10-15T03:00:43.965137758Z INFO: 10.32.3.1:57986 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:00:53.962+00:00 2026-10-15T03:00:53.964797296Z INFO: 10.32.3.1:35412 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:01:03.965+00:00 2026-10-15T03:01:03.965183315Z INFO: 10.32.3.1:41724 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:01:12.798+00:00 2026-10-15T03:01:12.794180811Z INFO: 127.0.0.1:45096 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:01:13.967+00:00 2026-10-15T03:01:13.964671642Z INFO: 10.32.3.1:51088 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:01:23.978+00:00 2026-10-15T03:01:23.965052643Z INFO: 10.32.3.1:42154 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl deleted file mode 100644 index 8d9562a1..00000000 --- a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T02:59:11.608+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T02:59:11.609+00:00", "t": 0.001, "phase": "create_model.start"} -{"ts": "2026-10-15T02:59:29.165+00:00", "t": 17.557, "phase": "create_model.done", "model_id": "cc688d59-8147-4315-9236-81ea9296fa1d"} -{"ts": "2026-10-15T02:59:32.362+00:00", "t": 20.753, "phase": "forward_backward.start"} -{"ts": "2026-10-15T02:59:35.480+00:00", "t": 23.872, "phase": "forward_backward.done"} -{"ts": "2026-10-15T02:59:36.339+00:00", "t": 24.731, "phase": "optim_step.done"} -{"ts": "2026-10-15T02:59:36.340+00:00", "t": 24.732, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T02:59:36.946+00:00", "t": 25.338, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T02:59:36.947+00:00", "t": 25.339, "phase": "sample.start"} -{"ts": "2026-10-15T03:01:23.638+00:00", "t": 132.029, "phase": "sample.done", "text": " 4"} -{"ts": "2026-10-15T03:01:23.639+00:00", "t": 132.03, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log deleted file mode 100644 index ebf20190..00000000 --- a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/client.log +++ /dev/null @@ -1,16 +0,0 @@ - Building open-rl-client @ file:///Users/sunilarora/work/open-rl/examples - Built open-rl-client @ file:///Users/sunilarora/work/open-rl/examples -Uninstalled 1 package in 1ms -Installed 1 package in 4ms -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.001s create_model.start -[cold-start] 17.557s create_model.done {'model_id': 'cc688d59-8147-4315-9236-81ea9296fa1d'} -[cold-start] 20.753s forward_backward.start -[cold-start] 23.872s forward_backward.done -[cold-start] 24.731s optim_step.done -[cold-start] 24.732s save_weights_and_get_sampling_client.start -[cold-start] 25.338s save_weights_and_get_sampling_client.done -[cold-start] 25.339s sample.start -[cold-start] 132.029s sample.done {'text': ' 4'} -[cold-start] 132.030s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt deleted file mode 100644 index eed48c3c..00000000 --- a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T03:01:46.957+00:00 diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json deleted file mode 100644 index b5ad8cd7..00000000 --- a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/events.json +++ /dev/null @@ -1,269 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:36Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176697983001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:36Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:36Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94ae008553cd", - "namespace": "openrl-system", - "resourceVersion": "1792033176815007016", - "uid": "363bd5d3-147c-4b0d-8500-d3a3b6232789" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:40Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176788543001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:40Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:40Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed139d233", - "namespace": "openrl-system", - "resourceVersion": "1792033180304687014", - "uid": "2463b9b8-32de-4c33-af86-ca945ab359e3" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:40Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176788543001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:40Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:40Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aed2f06544", - "namespace": "openrl-system", - "resourceVersion": "1792033180345279015", - "uid": "4766979d-5fb4-4934-96f7-b8e5c1833efa" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:40Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792033176788543001", - "uid": "358c385e-4cb7-4890-b522-cc6257e4c0a2" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:40Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:40Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de94aedddd6086", - "namespace": "openrl-system", - "resourceVersion": "1792033180517807014", - "uid": "e1c7e13f-7b29-4ed9-afdb-cc1a062a32d2" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153094671012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a883fadef0", - "namespace": "openrl-system", - "resourceVersion": "1792033153265231011", - "uid": "62c5cae6-2deb-43a0-a097-73157e760539" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:17Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153224047012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:17Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:17Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a9671f24f0", - "namespace": "openrl-system", - "resourceVersion": "1792033157084143008", - "uid": "9472795b-31b9-478f-a545-6146795d5503" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:17Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153224047012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:17Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:17Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a968b7d635", - "namespace": "openrl-system", - "resourceVersion": "1792033157103615024", - "uid": "11d1cf36-8464-42c3-94e2-1843723223c8" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T02:59:17Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792033153224047012", - "uid": "23e4198a-f0bf-48c8-b99a-93881e0d62f6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T02:59:17Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T02:59:17Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de94a973285d70", - "namespace": "openrl-system", - "resourceVersion": "1792033157264271007", - "uid": "8fa654a4-608f-4e8c-804f-9d1d5c91650a" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt deleted file mode 100644 index ba103934..00000000 --- a/runs/2026-10-15_cold_start/B-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt +++ /dev/null @@ -1,120 +0,0 @@ -2026-10-15T02:59:51.481521360Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T03:00:03.567266539Z INFO 10-15 03:00:03 [model.py:692] Resolved architecture: Qwen2ForCausalLM -2026-10-15T03:00:03.567589212Z INFO 10-15 03:00:03 [model.py:2030] Using max model len 8192 -2026-10-15T03:00:03.569678337Z INFO 10-15 03:00:03 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. -2026-10-15T03:00:03.982387755Z INFO 10-15 03:00:03 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T03:00:07.254113521Z WARNING 10-15 03:00:07 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T03:00:18.656636083Z (EngineCore pid=161) INFO 10-15 03:00:18 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T03:00:19.465208619Z (EngineCore pid=161) INFO 10-15 03:00:19 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_83c4163e04374cbaaa5cdd2b203e669a backend=nccl -2026-10-15T03:00:19.472365440Z (EngineCore pid=161) INFO 10-15 03:00:19 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T03:00:19.472394618Z (EngineCore pid=161) INFO 10-15 03:00:19 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T03:00:20.368364392Z (EngineCore pid=161) INFO 10-15 03:00:20 [model_runner.py:396] Loading model from scratch... -2026-10-15T03:00:21.228539711Z (EngineCore pid=161) INFO 10-15 03:00:21 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T03:00:21.228925613Z (EngineCore pid=161) INFO 10-15 03:00:21 [flash_attn.py:1116] Using FlashAttention version 2 -2026-10-15T03:00:21.343550869Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T03:00:21.343876865Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T03:00:21.715756916Z (EngineCore pid=161) INFO 10-15 03:00:21 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.61 GiB. -2026-10-15T03:00:21.715792572Z (EngineCore pid=161) INFO 10-15 03:00:21 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T03:00:21.716786271Z (EngineCore pid=161) -Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T02:59:23.867929492Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T02:59:23.867951058Z -2026-10-15T02:59:23.989922239Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... -2026-10-15T02:59:24.254380821Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T02:59:28.134128443Z -Loading weights: 0%| | 0/290 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T03:33:16.441826067Z (EngineCore pid=168) INFO 10-15 03:33:16 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c6e1717f2b3f4f1aad5a35495f331a54 backend=nccl -2026-10-15T03:33:16.446140127Z (EngineCore pid=168) INFO 10-15 03:33:16 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T03:33:16.446146945Z (EngineCore pid=168) INFO 10-15 03:33:16 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T03:33:17.209275505Z (EngineCore pid=168) INFO 10-15 03:33:17 [model_runner.py:396] Loading model from scratch... -2026-10-15T03:33:17.782392712Z (EngineCore pid=168) INFO 10-15 03:33:17 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T03:33:17.782713265Z (EngineCore pid=168) INFO 10-15 03:33:17 [flash_attn.py:1116] Using FlashAttention version 3 -2026-10-15T03:33:17.897956609Z (EngineCore pid=168) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T03:33:17.898105102Z (EngineCore pid=168) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T03:33:18.247383946Z (EngineCore pid=168) INFO 10-15 03:33:18 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. -2026-10-15T03:33:18.247405710Z (EngineCore pid=168) INFO 10-15 03:33:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T03:33:18.247843823Z (EngineCore pid=168) -Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T03:15:56.032450989Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T03:15:56.032474492Z -2026-10-15T03:15:56.267417499Z Loading base model Qwen/Qwen3-8B to cuda:0... -2026-10-15T03:15:56.394801806Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T03:30:59.475214316Z -Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 24h v1.35.8-gke.1380001 true -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 24h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json b/runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json deleted file mode 100644 index e1e02eb8..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-api-server-645ff9777d-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "645ff9777d", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - } - ], - "resourceVersion": "1791941173996783021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-79xdl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:02Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:26:02Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.10", - "podIPs": [ - { - "ip": "10.32.3.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T03:31:11Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-sampler", - "uid": "4637c9e3-a819-432d-9edb-58a1522564c4" - } - ], - "resourceVersion": "1792035171317391023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-sampler-4637c9e3" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "464502aa-adbd-4913-8de9-9cf72fa2e24d" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-4dt6k", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-4dt6k", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:32:51Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:31:11Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:32:51Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:32:51Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:31:11Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "40Gi" - }, - "containerID": "containerd://3427c91c85176d6732b6c0d7374ca8c171ff0233228cb15a8abf662d4ddfe519", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T03:32:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-4dt6k", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.1.7", - "podIPs": [ - { - "ip": "10.32.1.7" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T03:31:11Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T03:14:14Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-trainer", - "uid": "013a69b1-c398-4e81-a5c5-90edf5da837a" - } - ], - "resourceVersion": "1792034152554671006", - "uid": "5f7199d4-14aa-4747-9728-d940f5af679f" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-trainer-013a69b1" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "bbe9937b-9591-4fe5-9cd5-1845592a917d" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-qss8v", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-qss8v", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:15:52Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:14:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:15:52Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:15:52Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T03:14:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "36Gi" - }, - "containerID": "containerd://40c14afec9852a4e978707988d4e5ae0ddd5abd2a58fd807ce3b8e89eefe6dc6", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T03:15:51Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-qss8v", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.4.6", - "podIPs": [ - { - "ip": "10.32.4.6" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T03:14:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log b/runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log deleted file mode 100644 index e9451fdb..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/redis-ready.log +++ /dev/null @@ -1,1734 +0,0 @@ -2026-10-15T03:14:10.732+00:00 2026-10-15T03:14:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:11.737+00:00 2026-10-15T03:14:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:12.741+00:00 2026-10-15T03:14:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:13.746+00:00 2026-10-15T03:14:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:14.751+00:00 2026-10-15T03:14:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:15.757+00:00 2026-10-15T03:14:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:16.762+00:00 2026-10-15T03:14:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:17.766+00:00 2026-10-15T03:14:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:18.772+00:00 2026-10-15T03:14:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:19.779+00:00 2026-10-15T03:14:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:20.782+00:00 2026-10-15T03:14:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:21.787+00:00 2026-10-15T03:14:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:22.792+00:00 2026-10-15T03:14:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:23.797+00:00 2026-10-15T03:14:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:24.806+00:00 2026-10-15T03:14:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:25.807+00:00 2026-10-15T03:14:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:26.813+00:00 2026-10-15T03:14:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:27.816+00:00 2026-10-15T03:14:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:28.823+00:00 2026-10-15T03:14:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:29.837+00:00 2026-10-15T03:14:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:30.833+00:00 2026-10-15T03:14:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:31.838+00:00 2026-10-15T03:14:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:32.843+00:00 2026-10-15T03:14:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:33.848+00:00 2026-10-15T03:14:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:34.855+00:00 2026-10-15T03:14:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:35.860+00:00 2026-10-15T03:14:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:36.865+00:00 2026-10-15T03:14:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:37.870+00:00 2026-10-15T03:14:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:38.876+00:00 2026-10-15T03:14:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:39.883+00:00 2026-10-15T03:14:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:40.887+00:00 2026-10-15T03:14:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:41.893+00:00 2026-10-15T03:14:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:42.898+00:00 2026-10-15T03:14:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:43.903+00:00 2026-10-15T03:14:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:44.929+00:00 2026-10-15T03:14:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:45.914+00:00 2026-10-15T03:14:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:46.919+00:00 2026-10-15T03:14:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:47.924+00:00 2026-10-15T03:14:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:48.929+00:00 2026-10-15T03:14:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:50.075+00:00 2026-10-15T03:14:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:50.940+00:00 2026-10-15T03:14:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:51.946+00:00 2026-10-15T03:14:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:52.951+00:00 2026-10-15T03:14:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:53.956+00:00 2026-10-15T03:14:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:54.976+00:00 2026-10-15T03:14:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:55.969+00:00 2026-10-15T03:14:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:56.975+00:00 2026-10-15T03:14:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:57.981+00:00 2026-10-15T03:14:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:58.986+00:00 2026-10-15T03:14:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:14:59.990+00:00 2026-10-15T03:14:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:00.996+00:00 2026-10-15T03:15:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:02.003+00:00 2026-10-15T03:15:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:03.009+00:00 2026-10-15T03:15:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:04.014+00:00 2026-10-15T03:15:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:05.020+00:00 2026-10-15T03:15:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:06.026+00:00 2026-10-15T03:15:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:07.032+00:00 2026-10-15T03:15:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:08.038+00:00 2026-10-15T03:15:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:09.044+00:00 2026-10-15T03:15:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:10.051+00:00 2026-10-15T03:15:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:11.058+00:00 2026-10-15T03:15:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:12.062+00:00 2026-10-15T03:15:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:13.069+00:00 2026-10-15T03:15:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:14.090+00:00 2026-10-15T03:15:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:15.081+00:00 2026-10-15T03:15:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:16.084+00:00 2026-10-15T03:15:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:17.089+00:00 2026-10-15T03:15:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:18.131+00:00 2026-10-15T03:15:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:19.100+00:00 2026-10-15T03:15:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:20.106+00:00 2026-10-15T03:15:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:21.112+00:00 2026-10-15T03:15:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:22.118+00:00 2026-10-15T03:15:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:23.141+00:00 2026-10-15T03:15:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:24.130+00:00 2026-10-15T03:15:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:25.139+00:00 2026-10-15T03:15:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:26.143+00:00 2026-10-15T03:15:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:27.149+00:00 2026-10-15T03:15:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:28.166+00:00 2026-10-15T03:15:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:29.160+00:00 2026-10-15T03:15:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:30.166+00:00 2026-10-15T03:15:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:31.215+00:00 2026-10-15T03:15:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:32.177+00:00 2026-10-15T03:15:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:33.183+00:00 2026-10-15T03:15:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:34.188+00:00 2026-10-15T03:15:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:35.194+00:00 2026-10-15T03:15:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:36.201+00:00 2026-10-15T03:15:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:37.205+00:00 2026-10-15T03:15:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:38.211+00:00 2026-10-15T03:15:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:39.215+00:00 2026-10-15T03:15:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:40.221+00:00 2026-10-15T03:15:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:41.227+00:00 2026-10-15T03:15:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:42.232+00:00 2026-10-15T03:15:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:43.237+00:00 2026-10-15T03:15:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:44.243+00:00 2026-10-15T03:15:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:45.248+00:00 2026-10-15T03:15:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:46.254+00:00 2026-10-15T03:15:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:47.258+00:00 2026-10-15T03:15:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:48.264+00:00 2026-10-15T03:15:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:49.269+00:00 2026-10-15T03:15:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:50.279+00:00 2026-10-15T03:15:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:51.280+00:00 2026-10-15T03:15:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:52.285+00:00 2026-10-15T03:15:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:53.290+00:00 2026-10-15T03:15:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:54.297+00:00 2026-10-15T03:15:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:55.301+00:00 2026-10-15T03:15:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:56.306+00:00 2026-10-15T03:15:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:57.313+00:00 2026-10-15T03:15:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:58.320+00:00 2026-10-15T03:15:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:15:59.324+00:00 2026-10-15T03:15:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:00.327+00:00 2026-10-15T03:16:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:01.334+00:00 2026-10-15T03:16:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:02.340+00:00 2026-10-15T03:16:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:03.346+00:00 2026-10-15T03:16:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:04.351+00:00 2026-10-15T03:16:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:05.357+00:00 2026-10-15T03:16:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:06.361+00:00 2026-10-15T03:16:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:07.368+00:00 2026-10-15T03:16:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:08.372+00:00 2026-10-15T03:16:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:09.400+00:00 2026-10-15T03:16:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:10.382+00:00 2026-10-15T03:16:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:11.387+00:00 2026-10-15T03:16:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:12.392+00:00 2026-10-15T03:16:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:13.397+00:00 2026-10-15T03:16:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:14.402+00:00 2026-10-15T03:16:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:15.407+00:00 2026-10-15T03:16:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:16.413+00:00 2026-10-15T03:16:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:17.418+00:00 2026-10-15T03:16:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:18.422+00:00 2026-10-15T03:16:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:19.428+00:00 2026-10-15T03:16:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:20.433+00:00 2026-10-15T03:16:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:21.438+00:00 2026-10-15T03:16:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:22.444+00:00 2026-10-15T03:16:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:23.448+00:00 2026-10-15T03:16:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:24.454+00:00 2026-10-15T03:16:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:25.512+00:00 2026-10-15T03:16:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:26.466+00:00 2026-10-15T03:16:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:27.558+00:00 2026-10-15T03:16:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:28.481+00:00 2026-10-15T03:16:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:29.480+00:00 2026-10-15T03:16:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:30.485+00:00 2026-10-15T03:16:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:31.491+00:00 2026-10-15T03:16:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:32.498+00:00 2026-10-15T03:16:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:33.502+00:00 2026-10-15T03:16:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:34.508+00:00 2026-10-15T03:16:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:35.513+00:00 2026-10-15T03:16:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:36.525+00:00 2026-10-15T03:16:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:37.527+00:00 2026-10-15T03:16:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:38.529+00:00 2026-10-15T03:16:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:39.535+00:00 2026-10-15T03:16:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:40.539+00:00 2026-10-15T03:16:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:41.545+00:00 2026-10-15T03:16:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:42.551+00:00 2026-10-15T03:16:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:43.556+00:00 2026-10-15T03:16:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:44.562+00:00 2026-10-15T03:16:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:45.568+00:00 2026-10-15T03:16:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:46.574+00:00 2026-10-15T03:16:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:47.581+00:00 2026-10-15T03:16:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:48.594+00:00 2026-10-15T03:16:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:49.605+00:00 2026-10-15T03:16:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:50.595+00:00 2026-10-15T03:16:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:51.600+00:00 2026-10-15T03:16:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:52.612+00:00 2026-10-15T03:16:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:53.610+00:00 2026-10-15T03:16:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:54.617+00:00 2026-10-15T03:16:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:55.623+00:00 2026-10-15T03:16:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:56.629+00:00 2026-10-15T03:16:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:57.634+00:00 2026-10-15T03:16:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:58.639+00:00 2026-10-15T03:16:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:16:59.645+00:00 2026-10-15T03:16:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:00.650+00:00 2026-10-15T03:17:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:01.656+00:00 2026-10-15T03:17:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:02.661+00:00 2026-10-15T03:17:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:03.667+00:00 2026-10-15T03:17:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:04.673+00:00 2026-10-15T03:17:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:05.678+00:00 2026-10-15T03:17:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:06.683+00:00 2026-10-15T03:17:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:07.688+00:00 2026-10-15T03:17:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:08.693+00:00 2026-10-15T03:17:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:09.698+00:00 2026-10-15T03:17:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:10.703+00:00 2026-10-15T03:17:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:11.719+00:00 2026-10-15T03:17:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:12.714+00:00 2026-10-15T03:17:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:13.722+00:00 2026-10-15T03:17:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:14.738+00:00 2026-10-15T03:17:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:15.745+00:00 2026-10-15T03:17:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:16.736+00:00 2026-10-15T03:17:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:17.741+00:00 2026-10-15T03:17:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:18.746+00:00 2026-10-15T03:17:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:19.754+00:00 2026-10-15T03:17:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:20.758+00:00 2026-10-15T03:17:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:21.764+00:00 2026-10-15T03:17:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:22.768+00:00 2026-10-15T03:17:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:23.773+00:00 2026-10-15T03:17:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:24.778+00:00 2026-10-15T03:17:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:25.784+00:00 2026-10-15T03:17:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:26.789+00:00 2026-10-15T03:17:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:27.793+00:00 2026-10-15T03:17:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:28.801+00:00 2026-10-15T03:17:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:29.804+00:00 2026-10-15T03:17:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:30.810+00:00 2026-10-15T03:17:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:31.815+00:00 2026-10-15T03:17:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:32.822+00:00 2026-10-15T03:17:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:33.827+00:00 2026-10-15T03:17:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:34.835+00:00 2026-10-15T03:17:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:35.840+00:00 2026-10-15T03:17:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:36.845+00:00 2026-10-15T03:17:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:37.852+00:00 2026-10-15T03:17:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:38.857+00:00 2026-10-15T03:17:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:39.864+00:00 2026-10-15T03:17:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:40.870+00:00 2026-10-15T03:17:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:41.876+00:00 2026-10-15T03:17:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:42.889+00:00 2026-10-15T03:17:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:43.888+00:00 2026-10-15T03:17:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:44.894+00:00 2026-10-15T03:17:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:45.907+00:00 2026-10-15T03:17:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:46.904+00:00 2026-10-15T03:17:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:47.909+00:00 2026-10-15T03:17:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:48.914+00:00 2026-10-15T03:17:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:49.919+00:00 2026-10-15T03:17:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:50.925+00:00 2026-10-15T03:17:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:51.930+00:00 2026-10-15T03:17:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:52.963+00:00 2026-10-15T03:17:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:53.942+00:00 2026-10-15T03:17:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:54.947+00:00 2026-10-15T03:17:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:55.954+00:00 2026-10-15T03:17:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:56.956+00:00 2026-10-15T03:17:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:57.962+00:00 2026-10-15T03:17:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:58.966+00:00 2026-10-15T03:17:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:17:59.970+00:00 2026-10-15T03:17:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:00.985+00:00 2026-10-15T03:18:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:01.980+00:00 2026-10-15T03:18:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:02.986+00:00 2026-10-15T03:18:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:03.991+00:00 2026-10-15T03:18:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:05.001+00:00 2026-10-15T03:18:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:06.090+00:00 2026-10-15T03:18:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:07.008+00:00 2026-10-15T03:18:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:08.014+00:00 2026-10-15T03:18:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:09.019+00:00 2026-10-15T03:18:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:10.025+00:00 2026-10-15T03:18:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:11.029+00:00 2026-10-15T03:18:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:12.034+00:00 2026-10-15T03:18:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:13.039+00:00 2026-10-15T03:18:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:14.043+00:00 2026-10-15T03:18:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:15.049+00:00 2026-10-15T03:18:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:16.053+00:00 2026-10-15T03:18:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:17.058+00:00 2026-10-15T03:18:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:18.064+00:00 2026-10-15T03:18:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:19.071+00:00 2026-10-15T03:18:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:20.073+00:00 2026-10-15T03:18:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:21.079+00:00 2026-10-15T03:18:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:22.086+00:00 2026-10-15T03:18:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:23.091+00:00 2026-10-15T03:18:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:24.096+00:00 2026-10-15T03:18:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:25.107+00:00 2026-10-15T03:18:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:26.112+00:00 2026-10-15T03:18:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:27.116+00:00 2026-10-15T03:18:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:28.120+00:00 2026-10-15T03:18:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:29.124+00:00 2026-10-15T03:18:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:30.129+00:00 2026-10-15T03:18:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:31.135+00:00 2026-10-15T03:18:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:32.141+00:00 2026-10-15T03:18:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:33.186+00:00 2026-10-15T03:18:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:34.151+00:00 2026-10-15T03:18:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:35.157+00:00 2026-10-15T03:18:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:36.162+00:00 2026-10-15T03:18:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:37.167+00:00 2026-10-15T03:18:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:38.172+00:00 2026-10-15T03:18:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:39.183+00:00 2026-10-15T03:18:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:40.183+00:00 2026-10-15T03:18:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:41.187+00:00 2026-10-15T03:18:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:42.193+00:00 2026-10-15T03:18:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:43.198+00:00 2026-10-15T03:18:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:44.203+00:00 2026-10-15T03:18:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:45.207+00:00 2026-10-15T03:18:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:46.212+00:00 2026-10-15T03:18:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:47.217+00:00 2026-10-15T03:18:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:48.268+00:00 2026-10-15T03:18:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:49.228+00:00 2026-10-15T03:18:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:50.232+00:00 2026-10-15T03:18:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:51.237+00:00 2026-10-15T03:18:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:52.242+00:00 2026-10-15T03:18:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:53.247+00:00 2026-10-15T03:18:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:54.254+00:00 2026-10-15T03:18:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:55.258+00:00 2026-10-15T03:18:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:56.264+00:00 2026-10-15T03:18:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:57.268+00:00 2026-10-15T03:18:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:58.273+00:00 2026-10-15T03:18:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:18:59.279+00:00 2026-10-15T03:18:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:00.283+00:00 2026-10-15T03:19:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:01.288+00:00 2026-10-15T03:19:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:02.298+00:00 2026-10-15T03:19:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:03.299+00:00 2026-10-15T03:19:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:04.304+00:00 2026-10-15T03:19:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:05.308+00:00 2026-10-15T03:19:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:06.313+00:00 2026-10-15T03:19:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:07.318+00:00 2026-10-15T03:19:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:08.325+00:00 2026-10-15T03:19:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:09.329+00:00 2026-10-15T03:19:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:10.334+00:00 2026-10-15T03:19:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:11.341+00:00 2026-10-15T03:19:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:12.345+00:00 2026-10-15T03:19:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:13.350+00:00 2026-10-15T03:19:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:14.356+00:00 2026-10-15T03:19:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:15.363+00:00 2026-10-15T03:19:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:16.367+00:00 2026-10-15T03:19:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:17.371+00:00 2026-10-15T03:19:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:18.377+00:00 2026-10-15T03:19:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:19.382+00:00 2026-10-15T03:19:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:20.389+00:00 2026-10-15T03:19:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:21.395+00:00 2026-10-15T03:19:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:22.399+00:00 2026-10-15T03:19:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:23.404+00:00 2026-10-15T03:19:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:24.409+00:00 2026-10-15T03:19:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:25.414+00:00 2026-10-15T03:19:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:26.419+00:00 2026-10-15T03:19:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:27.424+00:00 2026-10-15T03:19:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:28.430+00:00 2026-10-15T03:19:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:29.437+00:00 2026-10-15T03:19:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:30.440+00:00 2026-10-15T03:19:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:31.445+00:00 2026-10-15T03:19:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:32.451+00:00 2026-10-15T03:19:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:33.459+00:00 2026-10-15T03:19:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:34.470+00:00 2026-10-15T03:19:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:35.469+00:00 2026-10-15T03:19:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:36.478+00:00 2026-10-15T03:19:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:37.483+00:00 2026-10-15T03:19:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:38.489+00:00 2026-10-15T03:19:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:39.491+00:00 2026-10-15T03:19:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:40.497+00:00 2026-10-15T03:19:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:41.501+00:00 2026-10-15T03:19:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:42.508+00:00 2026-10-15T03:19:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:43.533+00:00 2026-10-15T03:19:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:44.519+00:00 2026-10-15T03:19:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:45.523+00:00 2026-10-15T03:19:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:46.528+00:00 2026-10-15T03:19:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:47.533+00:00 2026-10-15T03:19:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:48.539+00:00 2026-10-15T03:19:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:49.544+00:00 2026-10-15T03:19:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:50.549+00:00 2026-10-15T03:19:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:51.554+00:00 2026-10-15T03:19:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:52.559+00:00 2026-10-15T03:19:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:53.564+00:00 2026-10-15T03:19:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:54.570+00:00 2026-10-15T03:19:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:55.575+00:00 2026-10-15T03:19:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:56.580+00:00 2026-10-15T03:19:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:57.584+00:00 2026-10-15T03:19:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:58.595+00:00 2026-10-15T03:19:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:19:59.594+00:00 2026-10-15T03:19:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:00.599+00:00 2026-10-15T03:20:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:01.611+00:00 2026-10-15T03:20:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:02.609+00:00 2026-10-15T03:20:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:03.615+00:00 2026-10-15T03:20:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:04.620+00:00 2026-10-15T03:20:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:05.642+00:00 2026-10-15T03:20:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:06.631+00:00 2026-10-15T03:20:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:07.634+00:00 2026-10-15T03:20:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:08.638+00:00 2026-10-15T03:20:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:09.643+00:00 2026-10-15T03:20:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:10.651+00:00 2026-10-15T03:20:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:11.653+00:00 2026-10-15T03:20:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:12.658+00:00 2026-10-15T03:20:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:13.663+00:00 2026-10-15T03:20:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:14.668+00:00 2026-10-15T03:20:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:15.674+00:00 2026-10-15T03:20:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:16.678+00:00 2026-10-15T03:20:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:17.683+00:00 2026-10-15T03:20:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:18.688+00:00 2026-10-15T03:20:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:19.693+00:00 2026-10-15T03:20:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:20.705+00:00 2026-10-15T03:20:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:21.712+00:00 2026-10-15T03:20:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:22.707+00:00 2026-10-15T03:20:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:23.712+00:00 2026-10-15T03:20:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:24.718+00:00 2026-10-15T03:20:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:25.721+00:00 2026-10-15T03:20:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:26.726+00:00 2026-10-15T03:20:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:27.731+00:00 2026-10-15T03:20:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:28.742+00:00 2026-10-15T03:20:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:29.742+00:00 2026-10-15T03:20:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:30.748+00:00 2026-10-15T03:20:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:31.751+00:00 2026-10-15T03:20:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:32.756+00:00 2026-10-15T03:20:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:33.762+00:00 2026-10-15T03:20:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:34.766+00:00 2026-10-15T03:20:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:35.787+00:00 2026-10-15T03:20:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:36.776+00:00 2026-10-15T03:20:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:37.780+00:00 2026-10-15T03:20:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:38.791+00:00 2026-10-15T03:20:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:39.790+00:00 2026-10-15T03:20:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:40.796+00:00 2026-10-15T03:20:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:41.800+00:00 2026-10-15T03:20:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:42.804+00:00 2026-10-15T03:20:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:43.809+00:00 2026-10-15T03:20:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:44.814+00:00 2026-10-15T03:20:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:45.819+00:00 2026-10-15T03:20:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:46.833+00:00 2026-10-15T03:20:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:47.830+00:00 2026-10-15T03:20:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:48.836+00:00 2026-10-15T03:20:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:49.846+00:00 2026-10-15T03:20:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:50.846+00:00 2026-10-15T03:20:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:51.850+00:00 2026-10-15T03:20:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:52.855+00:00 2026-10-15T03:20:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:53.860+00:00 2026-10-15T03:20:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:54.874+00:00 2026-10-15T03:20:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:55.870+00:00 2026-10-15T03:20:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:56.912+00:00 2026-10-15T03:20:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:57.881+00:00 2026-10-15T03:20:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:58.891+00:00 2026-10-15T03:20:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:20:59.891+00:00 2026-10-15T03:20:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:00.896+00:00 2026-10-15T03:21:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:01.901+00:00 2026-10-15T03:21:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:02.907+00:00 2026-10-15T03:21:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:03.912+00:00 2026-10-15T03:21:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:04.921+00:00 2026-10-15T03:21:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:05.926+00:00 2026-10-15T03:21:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:06.934+00:00 2026-10-15T03:21:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:07.938+00:00 2026-10-15T03:21:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:08.951+00:00 2026-10-15T03:21:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:09.948+00:00 2026-10-15T03:21:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:10.953+00:00 2026-10-15T03:21:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:11.958+00:00 2026-10-15T03:21:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:12.963+00:00 2026-10-15T03:21:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:13.968+00:00 2026-10-15T03:21:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:14.973+00:00 2026-10-15T03:21:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:15.979+00:00 2026-10-15T03:21:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:16.997+00:00 2026-10-15T03:21:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:17.989+00:00 2026-10-15T03:21:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:19.013+00:00 2026-10-15T03:21:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:20.001+00:00 2026-10-15T03:21:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:21.005+00:00 2026-10-15T03:21:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:22.011+00:00 2026-10-15T03:21:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:23.018+00:00 2026-10-15T03:21:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:24.113+00:00 2026-10-15T03:21:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:25.029+00:00 2026-10-15T03:21:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:26.033+00:00 2026-10-15T03:21:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:27.040+00:00 2026-10-15T03:21:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:28.055+00:00 2026-10-15T03:21:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:29.156+00:00 2026-10-15T03:21:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:30.056+00:00 2026-10-15T03:21:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:31.075+00:00 2026-10-15T03:21:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:32.066+00:00 2026-10-15T03:21:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:33.071+00:00 2026-10-15T03:21:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:34.077+00:00 2026-10-15T03:21:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:35.084+00:00 2026-10-15T03:21:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:36.091+00:00 2026-10-15T03:21:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:37.094+00:00 2026-10-15T03:21:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:38.099+00:00 2026-10-15T03:21:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:39.105+00:00 2026-10-15T03:21:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:40.112+00:00 2026-10-15T03:21:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:41.118+00:00 2026-10-15T03:21:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:42.121+00:00 2026-10-15T03:21:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:43.129+00:00 2026-10-15T03:21:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:44.135+00:00 2026-10-15T03:21:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:45.140+00:00 2026-10-15T03:21:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:46.144+00:00 2026-10-15T03:21:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:47.150+00:00 2026-10-15T03:21:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:48.155+00:00 2026-10-15T03:21:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:49.159+00:00 2026-10-15T03:21:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:50.164+00:00 2026-10-15T03:21:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:51.169+00:00 2026-10-15T03:21:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:52.178+00:00 2026-10-15T03:21:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:53.180+00:00 2026-10-15T03:21:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:54.185+00:00 2026-10-15T03:21:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:55.190+00:00 2026-10-15T03:21:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:56.200+00:00 2026-10-15T03:21:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:57.200+00:00 2026-10-15T03:21:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:58.205+00:00 2026-10-15T03:21:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:21:59.209+00:00 2026-10-15T03:21:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:00.214+00:00 2026-10-15T03:22:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:01.249+00:00 2026-10-15T03:22:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:02.224+00:00 2026-10-15T03:22:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:03.232+00:00 2026-10-15T03:22:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:04.242+00:00 2026-10-15T03:22:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:05.248+00:00 2026-10-15T03:22:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:06.251+00:00 2026-10-15T03:22:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:07.251+00:00 2026-10-15T03:22:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:08.255+00:00 2026-10-15T03:22:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:09.260+00:00 2026-10-15T03:22:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:10.270+00:00 2026-10-15T03:22:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:11.276+00:00 2026-10-15T03:22:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:12.278+00:00 2026-10-15T03:22:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:13.283+00:00 2026-10-15T03:22:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:14.289+00:00 2026-10-15T03:22:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:15.294+00:00 2026-10-15T03:22:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:16.298+00:00 2026-10-15T03:22:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:17.304+00:00 2026-10-15T03:22:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:18.309+00:00 2026-10-15T03:22:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:19.314+00:00 2026-10-15T03:22:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:20.321+00:00 2026-10-15T03:22:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:21.325+00:00 2026-10-15T03:22:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:22.332+00:00 2026-10-15T03:22:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:23.337+00:00 2026-10-15T03:22:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:24.341+00:00 2026-10-15T03:22:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:25.349+00:00 2026-10-15T03:22:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:26.352+00:00 2026-10-15T03:22:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:27.358+00:00 2026-10-15T03:22:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:28.362+00:00 2026-10-15T03:22:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:29.368+00:00 2026-10-15T03:22:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:30.468+00:00 2026-10-15T03:22:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:31.380+00:00 2026-10-15T03:22:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:32.385+00:00 2026-10-15T03:22:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:33.391+00:00 2026-10-15T03:22:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:34.396+00:00 2026-10-15T03:22:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:35.400+00:00 2026-10-15T03:22:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:36.406+00:00 2026-10-15T03:22:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:37.410+00:00 2026-10-15T03:22:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:38.422+00:00 2026-10-15T03:22:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:39.421+00:00 2026-10-15T03:22:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:40.427+00:00 2026-10-15T03:22:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:41.431+00:00 2026-10-15T03:22:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:42.435+00:00 2026-10-15T03:22:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:43.441+00:00 2026-10-15T03:22:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:44.445+00:00 2026-10-15T03:22:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:45.481+00:00 2026-10-15T03:22:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:46.457+00:00 2026-10-15T03:22:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:47.461+00:00 2026-10-15T03:22:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:48.467+00:00 2026-10-15T03:22:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:49.471+00:00 2026-10-15T03:22:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:50.477+00:00 2026-10-15T03:22:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:51.483+00:00 2026-10-15T03:22:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:52.488+00:00 2026-10-15T03:22:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:53.492+00:00 2026-10-15T03:22:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:54.499+00:00 2026-10-15T03:22:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:55.503+00:00 2026-10-15T03:22:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:56.507+00:00 2026-10-15T03:22:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:57.512+00:00 2026-10-15T03:22:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:58.517+00:00 2026-10-15T03:22:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:22:59.533+00:00 2026-10-15T03:22:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:00.528+00:00 2026-10-15T03:23:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:01.533+00:00 2026-10-15T03:23:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:02.539+00:00 2026-10-15T03:23:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:03.548+00:00 2026-10-15T03:23:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:04.549+00:00 2026-10-15T03:23:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:05.557+00:00 2026-10-15T03:23:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:06.559+00:00 2026-10-15T03:23:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:07.573+00:00 2026-10-15T03:23:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:08.570+00:00 2026-10-15T03:23:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:09.575+00:00 2026-10-15T03:23:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:10.581+00:00 2026-10-15T03:23:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:11.584+00:00 2026-10-15T03:23:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:12.590+00:00 2026-10-15T03:23:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:13.595+00:00 2026-10-15T03:23:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:14.599+00:00 2026-10-15T03:23:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:15.604+00:00 2026-10-15T03:23:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:16.609+00:00 2026-10-15T03:23:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:17.614+00:00 2026-10-15T03:23:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:18.751+00:00 2026-10-15T03:23:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:19.624+00:00 2026-10-15T03:23:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:20.629+00:00 2026-10-15T03:23:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:21.669+00:00 2026-10-15T03:23:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:22.641+00:00 2026-10-15T03:23:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:23.658+00:00 2026-10-15T03:23:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:24.650+00:00 2026-10-15T03:23:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:25.656+00:00 2026-10-15T03:23:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:26.661+00:00 2026-10-15T03:23:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:27.665+00:00 2026-10-15T03:23:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:28.672+00:00 2026-10-15T03:23:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:29.741+00:00 2026-10-15T03:23:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:30.681+00:00 2026-10-15T03:23:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:31.687+00:00 2026-10-15T03:23:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:32.696+00:00 2026-10-15T03:23:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:33.702+00:00 2026-10-15T03:23:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:34.702+00:00 2026-10-15T03:23:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:35.708+00:00 2026-10-15T03:23:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:36.714+00:00 2026-10-15T03:23:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:37.720+00:00 2026-10-15T03:23:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:38.723+00:00 2026-10-15T03:23:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:39.728+00:00 2026-10-15T03:23:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:40.734+00:00 2026-10-15T03:23:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:41.739+00:00 2026-10-15T03:23:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:42.745+00:00 2026-10-15T03:23:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:43.750+00:00 2026-10-15T03:23:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:44.758+00:00 2026-10-15T03:23:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:45.762+00:00 2026-10-15T03:23:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:46.766+00:00 2026-10-15T03:23:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:47.810+00:00 2026-10-15T03:23:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:48.777+00:00 2026-10-15T03:23:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:49.783+00:00 2026-10-15T03:23:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:50.787+00:00 2026-10-15T03:23:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:51.791+00:00 2026-10-15T03:23:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:52.801+00:00 2026-10-15T03:23:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:53.801+00:00 2026-10-15T03:23:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:54.805+00:00 2026-10-15T03:23:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:55.817+00:00 2026-10-15T03:23:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:56.816+00:00 2026-10-15T03:23:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:57.827+00:00 2026-10-15T03:23:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:58.826+00:00 2026-10-15T03:23:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:23:59.833+00:00 2026-10-15T03:23:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:00.836+00:00 2026-10-15T03:24:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:01.842+00:00 2026-10-15T03:24:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:02.854+00:00 2026-10-15T03:24:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:03.856+00:00 2026-10-15T03:24:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:04.860+00:00 2026-10-15T03:24:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:05.866+00:00 2026-10-15T03:24:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:06.872+00:00 2026-10-15T03:24:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:07.879+00:00 2026-10-15T03:24:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:08.883+00:00 2026-10-15T03:24:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:09.889+00:00 2026-10-15T03:24:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:10.893+00:00 2026-10-15T03:24:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:11.903+00:00 2026-10-15T03:24:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:12.904+00:00 2026-10-15T03:24:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:14.099+00:00 2026-10-15T03:24:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:14.918+00:00 2026-10-15T03:24:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:15.920+00:00 2026-10-15T03:24:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:16.967+00:00 2026-10-15T03:24:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:17.929+00:00 2026-10-15T03:24:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:18.934+00:00 2026-10-15T03:24:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:19.939+00:00 2026-10-15T03:24:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:20.945+00:00 2026-10-15T03:24:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:21.958+00:00 2026-10-15T03:24:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:22.955+00:00 2026-10-15T03:24:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:23.960+00:00 2026-10-15T03:24:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:24.965+00:00 2026-10-15T03:24:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:25.970+00:00 2026-10-15T03:24:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:26.975+00:00 2026-10-15T03:24:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:27.979+00:00 2026-10-15T03:24:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:28.984+00:00 2026-10-15T03:24:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:29.989+00:00 2026-10-15T03:24:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:30.995+00:00 2026-10-15T03:24:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:32.010+00:00 2026-10-15T03:24:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:33.005+00:00 2026-10-15T03:24:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:34.021+00:00 2026-10-15T03:24:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:35.015+00:00 2026-10-15T03:24:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:36.021+00:00 2026-10-15T03:24:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:37.025+00:00 2026-10-15T03:24:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:38.033+00:00 2026-10-15T03:24:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:39.058+00:00 2026-10-15T03:24:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:40.041+00:00 2026-10-15T03:24:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:41.046+00:00 2026-10-15T03:24:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:42.061+00:00 2026-10-15T03:24:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:43.057+00:00 2026-10-15T03:24:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:44.066+00:00 2026-10-15T03:24:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:45.066+00:00 2026-10-15T03:24:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:46.071+00:00 2026-10-15T03:24:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:47.078+00:00 2026-10-15T03:24:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:48.082+00:00 2026-10-15T03:24:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:49.086+00:00 2026-10-15T03:24:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:50.092+00:00 2026-10-15T03:24:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:51.096+00:00 2026-10-15T03:24:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:52.102+00:00 2026-10-15T03:24:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:53.108+00:00 2026-10-15T03:24:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:54.113+00:00 2026-10-15T03:24:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:55.119+00:00 2026-10-15T03:24:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:56.132+00:00 2026-10-15T03:24:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:57.144+00:00 2026-10-15T03:24:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:58.140+00:00 2026-10-15T03:24:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:24:59.145+00:00 2026-10-15T03:24:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:00.150+00:00 2026-10-15T03:25:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:01.158+00:00 2026-10-15T03:25:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:02.159+00:00 2026-10-15T03:25:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:03.255+00:00 2026-10-15T03:25:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:04.170+00:00 2026-10-15T03:25:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:05.175+00:00 2026-10-15T03:25:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:06.339+00:00 2026-10-15T03:25:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:07.185+00:00 2026-10-15T03:25:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:08.194+00:00 2026-10-15T03:25:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:09.195+00:00 2026-10-15T03:25:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:10.214+00:00 2026-10-15T03:25:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:11.205+00:00 2026-10-15T03:25:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:12.211+00:00 2026-10-15T03:25:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:13.216+00:00 2026-10-15T03:25:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:14.220+00:00 2026-10-15T03:25:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:15.229+00:00 2026-10-15T03:25:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:16.231+00:00 2026-10-15T03:25:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:17.238+00:00 2026-10-15T03:25:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:18.241+00:00 2026-10-15T03:25:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:19.252+00:00 2026-10-15T03:25:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:20.252+00:00 2026-10-15T03:25:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:21.257+00:00 2026-10-15T03:25:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:22.262+00:00 2026-10-15T03:25:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:23.266+00:00 2026-10-15T03:25:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:24.271+00:00 2026-10-15T03:25:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:25.373+00:00 2026-10-15T03:25:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:26.282+00:00 2026-10-15T03:25:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:27.297+00:00 2026-10-15T03:25:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:28.292+00:00 2026-10-15T03:25:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:29.298+00:00 2026-10-15T03:25:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:30.302+00:00 2026-10-15T03:25:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:31.306+00:00 2026-10-15T03:25:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:32.311+00:00 2026-10-15T03:25:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:33.318+00:00 2026-10-15T03:25:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:34.322+00:00 2026-10-15T03:25:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:35.331+00:00 2026-10-15T03:25:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:36.331+00:00 2026-10-15T03:25:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:37.336+00:00 2026-10-15T03:25:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:38.341+00:00 2026-10-15T03:25:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:39.345+00:00 2026-10-15T03:25:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:40.350+00:00 2026-10-15T03:25:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:41.355+00:00 2026-10-15T03:25:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:42.360+00:00 2026-10-15T03:25:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:43.365+00:00 2026-10-15T03:25:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:44.370+00:00 2026-10-15T03:25:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:45.381+00:00 2026-10-15T03:25:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:46.379+00:00 2026-10-15T03:25:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:47.384+00:00 2026-10-15T03:25:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:48.389+00:00 2026-10-15T03:25:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:49.393+00:00 2026-10-15T03:25:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:50.398+00:00 2026-10-15T03:25:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:51.402+00:00 2026-10-15T03:25:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:52.407+00:00 2026-10-15T03:25:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:53.411+00:00 2026-10-15T03:25:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:54.415+00:00 2026-10-15T03:25:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:55.421+00:00 2026-10-15T03:25:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:56.426+00:00 2026-10-15T03:25:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:57.431+00:00 2026-10-15T03:25:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:58.436+00:00 2026-10-15T03:25:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:25:59.441+00:00 2026-10-15T03:25:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:00.446+00:00 2026-10-15T03:26:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:01.452+00:00 2026-10-15T03:26:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:02.456+00:00 2026-10-15T03:26:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:03.461+00:00 2026-10-15T03:26:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:04.467+00:00 2026-10-15T03:26:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:05.471+00:00 2026-10-15T03:26:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:06.476+00:00 2026-10-15T03:26:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:07.481+00:00 2026-10-15T03:26:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:08.485+00:00 2026-10-15T03:26:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:09.490+00:00 2026-10-15T03:26:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:10.495+00:00 2026-10-15T03:26:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:11.500+00:00 2026-10-15T03:26:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:12.508+00:00 2026-10-15T03:26:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:13.510+00:00 2026-10-15T03:26:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:14.515+00:00 2026-10-15T03:26:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:15.528+00:00 2026-10-15T03:26:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:16.524+00:00 2026-10-15T03:26:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:17.529+00:00 2026-10-15T03:26:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:18.533+00:00 2026-10-15T03:26:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:19.538+00:00 2026-10-15T03:26:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:20.544+00:00 2026-10-15T03:26:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:21.549+00:00 2026-10-15T03:26:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:22.554+00:00 2026-10-15T03:26:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:23.558+00:00 2026-10-15T03:26:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:24.563+00:00 2026-10-15T03:26:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:25.570+00:00 2026-10-15T03:26:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:26.573+00:00 2026-10-15T03:26:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:27.654+00:00 2026-10-15T03:26:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:28.583+00:00 2026-10-15T03:26:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:29.588+00:00 2026-10-15T03:26:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:30.593+00:00 2026-10-15T03:26:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:31.598+00:00 2026-10-15T03:26:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:32.622+00:00 2026-10-15T03:26:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:33.610+00:00 2026-10-15T03:26:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:34.616+00:00 2026-10-15T03:26:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:35.624+00:00 2026-10-15T03:26:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:36.625+00:00 2026-10-15T03:26:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:37.635+00:00 2026-10-15T03:26:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:38.635+00:00 2026-10-15T03:26:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:39.641+00:00 2026-10-15T03:26:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:40.644+00:00 2026-10-15T03:26:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:41.651+00:00 2026-10-15T03:26:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:42.655+00:00 2026-10-15T03:26:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:43.661+00:00 2026-10-15T03:26:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:44.665+00:00 2026-10-15T03:26:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:45.670+00:00 2026-10-15T03:26:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:46.674+00:00 2026-10-15T03:26:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:47.679+00:00 2026-10-15T03:26:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:48.685+00:00 2026-10-15T03:26:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:49.693+00:00 2026-10-15T03:26:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:50.695+00:00 2026-10-15T03:26:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:51.701+00:00 2026-10-15T03:26:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:52.708+00:00 2026-10-15T03:26:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:53.712+00:00 2026-10-15T03:26:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:54.715+00:00 2026-10-15T03:26:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:55.724+00:00 2026-10-15T03:26:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:56.726+00:00 2026-10-15T03:26:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:57.732+00:00 2026-10-15T03:26:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:58.736+00:00 2026-10-15T03:26:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:26:59.742+00:00 2026-10-15T03:26:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:00.746+00:00 2026-10-15T03:27:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:01.836+00:00 2026-10-15T03:27:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:02.756+00:00 2026-10-15T03:27:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:03.763+00:00 2026-10-15T03:27:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:04.768+00:00 2026-10-15T03:27:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:05.772+00:00 2026-10-15T03:27:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:06.778+00:00 2026-10-15T03:27:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:07.783+00:00 2026-10-15T03:27:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:08.790+00:00 2026-10-15T03:27:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:09.792+00:00 2026-10-15T03:27:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:10.797+00:00 2026-10-15T03:27:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:11.801+00:00 2026-10-15T03:27:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:12.808+00:00 2026-10-15T03:27:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:13.813+00:00 2026-10-15T03:27:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:14.816+00:00 2026-10-15T03:27:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:15.821+00:00 2026-10-15T03:27:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:16.826+00:00 2026-10-15T03:27:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:17.831+00:00 2026-10-15T03:27:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:18.837+00:00 2026-10-15T03:27:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:19.843+00:00 2026-10-15T03:27:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:20.847+00:00 2026-10-15T03:27:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:21.855+00:00 2026-10-15T03:27:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:22.857+00:00 2026-10-15T03:27:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:23.862+00:00 2026-10-15T03:27:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:24.867+00:00 2026-10-15T03:27:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:25.872+00:00 2026-10-15T03:27:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:26.877+00:00 2026-10-15T03:27:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:27.887+00:00 2026-10-15T03:27:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:28.888+00:00 2026-10-15T03:27:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:29.896+00:00 2026-10-15T03:27:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:30.898+00:00 2026-10-15T03:27:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:31.904+00:00 2026-10-15T03:27:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:32.909+00:00 2026-10-15T03:27:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:33.914+00:00 2026-10-15T03:27:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:34.921+00:00 2026-10-15T03:27:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:35.927+00:00 2026-10-15T03:27:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:36.932+00:00 2026-10-15T03:27:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:37.938+00:00 2026-10-15T03:27:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:38.943+00:00 2026-10-15T03:27:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:39.947+00:00 2026-10-15T03:27:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:40.954+00:00 2026-10-15T03:27:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:41.957+00:00 2026-10-15T03:27:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:42.963+00:00 2026-10-15T03:27:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:43.969+00:00 2026-10-15T03:27:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:44.973+00:00 2026-10-15T03:27:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:45.978+00:00 2026-10-15T03:27:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:46.991+00:00 2026-10-15T03:27:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:47.998+00:00 2026-10-15T03:27:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:48.996+00:00 2026-10-15T03:27:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:50.001+00:00 2026-10-15T03:27:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:51.022+00:00 2026-10-15T03:27:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:52.011+00:00 2026-10-15T03:27:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:53.014+00:00 2026-10-15T03:27:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:54.020+00:00 2026-10-15T03:27:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:55.024+00:00 2026-10-15T03:27:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:56.029+00:00 2026-10-15T03:27:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:57.034+00:00 2026-10-15T03:27:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:58.040+00:00 2026-10-15T03:27:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:27:59.180+00:00 2026-10-15T03:27:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:00.060+00:00 2026-10-15T03:28:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:01.137+00:00 2026-10-15T03:28:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:02.060+00:00 2026-10-15T03:28:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:03.064+00:00 2026-10-15T03:28:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:04.069+00:00 2026-10-15T03:28:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:05.074+00:00 2026-10-15T03:28:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:06.148+00:00 2026-10-15T03:28:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:07.084+00:00 2026-10-15T03:28:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:08.089+00:00 2026-10-15T03:28:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:09.094+00:00 2026-10-15T03:28:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:10.101+00:00 2026-10-15T03:28:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:11.264+00:00 2026-10-15T03:28:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:12.115+00:00 2026-10-15T03:28:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:13.118+00:00 2026-10-15T03:28:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:14.124+00:00 2026-10-15T03:28:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:15.129+00:00 2026-10-15T03:28:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:16.135+00:00 2026-10-15T03:28:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:17.140+00:00 2026-10-15T03:28:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:18.145+00:00 2026-10-15T03:28:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:19.150+00:00 2026-10-15T03:28:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:20.162+00:00 2026-10-15T03:28:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:21.163+00:00 2026-10-15T03:28:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:22.166+00:00 2026-10-15T03:28:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:23.170+00:00 2026-10-15T03:28:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:24.176+00:00 2026-10-15T03:28:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:25.181+00:00 2026-10-15T03:28:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:26.187+00:00 2026-10-15T03:28:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:27.196+00:00 2026-10-15T03:28:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:28.197+00:00 2026-10-15T03:28:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:29.202+00:00 2026-10-15T03:28:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:30.209+00:00 2026-10-15T03:28:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:31.214+00:00 2026-10-15T03:28:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:32.221+00:00 2026-10-15T03:28:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:33.225+00:00 2026-10-15T03:28:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:34.230+00:00 2026-10-15T03:28:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:35.235+00:00 2026-10-15T03:28:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:36.242+00:00 2026-10-15T03:28:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:37.248+00:00 2026-10-15T03:28:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:38.253+00:00 2026-10-15T03:28:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:39.264+00:00 2026-10-15T03:28:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:40.264+00:00 2026-10-15T03:28:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:41.270+00:00 2026-10-15T03:28:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:42.274+00:00 2026-10-15T03:28:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:43.418+00:00 2026-10-15T03:28:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:44.284+00:00 2026-10-15T03:28:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:45.289+00:00 2026-10-15T03:28:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:46.294+00:00 2026-10-15T03:28:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:47.300+00:00 2026-10-15T03:28:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:48.304+00:00 2026-10-15T03:28:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:49.309+00:00 2026-10-15T03:28:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:50.314+00:00 2026-10-15T03:28:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:51.321+00:00 2026-10-15T03:28:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:52.325+00:00 2026-10-15T03:28:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:53.330+00:00 2026-10-15T03:28:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:54.334+00:00 2026-10-15T03:28:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:55.340+00:00 2026-10-15T03:28:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:56.345+00:00 2026-10-15T03:28:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:57.349+00:00 2026-10-15T03:28:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:58.354+00:00 2026-10-15T03:28:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:28:59.358+00:00 2026-10-15T03:28:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:00.364+00:00 2026-10-15T03:29:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:01.370+00:00 2026-10-15T03:29:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:02.374+00:00 2026-10-15T03:29:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:03.380+00:00 2026-10-15T03:29:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:04.385+00:00 2026-10-15T03:29:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:05.389+00:00 2026-10-15T03:29:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:06.395+00:00 2026-10-15T03:29:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:07.401+00:00 2026-10-15T03:29:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:08.404+00:00 2026-10-15T03:29:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:09.409+00:00 2026-10-15T03:29:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:10.414+00:00 2026-10-15T03:29:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:11.421+00:00 2026-10-15T03:29:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:12.424+00:00 2026-10-15T03:29:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:13.429+00:00 2026-10-15T03:29:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:14.436+00:00 2026-10-15T03:29:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:15.439+00:00 2026-10-15T03:29:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:16.445+00:00 2026-10-15T03:29:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:17.452+00:00 2026-10-15T03:29:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:18.461+00:00 2026-10-15T03:29:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:19.465+00:00 2026-10-15T03:29:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:20.465+00:00 2026-10-15T03:29:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:21.471+00:00 2026-10-15T03:29:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:22.476+00:00 2026-10-15T03:29:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:23.481+00:00 2026-10-15T03:29:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:24.485+00:00 2026-10-15T03:29:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:25.489+00:00 2026-10-15T03:29:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:26.501+00:00 2026-10-15T03:29:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:27.510+00:00 2026-10-15T03:29:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:28.503+00:00 2026-10-15T03:29:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:29.509+00:00 2026-10-15T03:29:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:30.514+00:00 2026-10-15T03:29:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:31.518+00:00 2026-10-15T03:29:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:32.523+00:00 2026-10-15T03:29:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:33.527+00:00 2026-10-15T03:29:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:34.532+00:00 2026-10-15T03:29:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:35.537+00:00 2026-10-15T03:29:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:36.546+00:00 2026-10-15T03:29:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:37.546+00:00 2026-10-15T03:29:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:38.557+00:00 2026-10-15T03:29:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:39.584+00:00 2026-10-15T03:29:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:40.572+00:00 2026-10-15T03:29:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:41.569+00:00 2026-10-15T03:29:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:42.571+00:00 2026-10-15T03:29:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:43.576+00:00 2026-10-15T03:29:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:44.600+00:00 2026-10-15T03:29:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:45.586+00:00 2026-10-15T03:29:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:46.592+00:00 2026-10-15T03:29:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:47.596+00:00 2026-10-15T03:29:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:48.601+00:00 2026-10-15T03:29:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:49.609+00:00 2026-10-15T03:29:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:50.612+00:00 2026-10-15T03:29:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:51.617+00:00 2026-10-15T03:29:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:52.622+00:00 2026-10-15T03:29:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:53.639+00:00 2026-10-15T03:29:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:54.633+00:00 2026-10-15T03:29:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:55.637+00:00 2026-10-15T03:29:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:56.645+00:00 2026-10-15T03:29:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:57.646+00:00 2026-10-15T03:29:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:58.652+00:00 2026-10-15T03:29:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:29:59.656+00:00 2026-10-15T03:29:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:00.672+00:00 2026-10-15T03:30:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:01.668+00:00 2026-10-15T03:30:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:02.675+00:00 2026-10-15T03:30:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:03.677+00:00 2026-10-15T03:30:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:04.683+00:00 2026-10-15T03:30:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:05.688+00:00 2026-10-15T03:30:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:06.694+00:00 2026-10-15T03:30:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:07.700+00:00 2026-10-15T03:30:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:08.820+00:00 2026-10-15T03:30:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:09.709+00:00 2026-10-15T03:30:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:10.715+00:00 2026-10-15T03:30:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:11.720+00:00 2026-10-15T03:30:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:12.726+00:00 2026-10-15T03:30:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:13.730+00:00 2026-10-15T03:30:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:14.736+00:00 2026-10-15T03:30:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:15.741+00:00 2026-10-15T03:30:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:16.746+00:00 2026-10-15T03:30:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:17.752+00:00 2026-10-15T03:30:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:18.755+00:00 2026-10-15T03:30:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:19.799+00:00 2026-10-15T03:30:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:20.767+00:00 2026-10-15T03:30:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:21.772+00:00 2026-10-15T03:30:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:22.777+00:00 2026-10-15T03:30:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:23.782+00:00 2026-10-15T03:30:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:24.788+00:00 2026-10-15T03:30:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:25.791+00:00 2026-10-15T03:30:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:26.796+00:00 2026-10-15T03:30:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:27.801+00:00 2026-10-15T03:30:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:28.806+00:00 2026-10-15T03:30:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:29.812+00:00 2026-10-15T03:30:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:30.819+00:00 2026-10-15T03:30:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:31.826+00:00 2026-10-15T03:30:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:32.827+00:00 2026-10-15T03:30:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:33.832+00:00 2026-10-15T03:30:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:34.838+00:00 2026-10-15T03:30:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:35.844+00:00 2026-10-15T03:30:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:36.848+00:00 2026-10-15T03:30:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:37.853+00:00 2026-10-15T03:30:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:38.859+00:00 2026-10-15T03:30:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:39.954+00:00 2026-10-15T03:30:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:40.869+00:00 2026-10-15T03:30:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:41.876+00:00 2026-10-15T03:30:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:42.879+00:00 2026-10-15T03:30:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:43.884+00:00 2026-10-15T03:30:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:44.889+00:00 2026-10-15T03:30:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:45.893+00:00 2026-10-15T03:30:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:46.913+00:00 2026-10-15T03:30:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:47.902+00:00 2026-10-15T03:30:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:48.907+00:00 2026-10-15T03:30:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:49.914+00:00 2026-10-15T03:30:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:50.919+00:00 2026-10-15T03:30:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:51.923+00:00 2026-10-15T03:30:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:52.928+00:00 2026-10-15T03:30:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:53.933+00:00 2026-10-15T03:30:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:54.937+00:00 2026-10-15T03:30:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:55.943+00:00 2026-10-15T03:30:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:56.947+00:00 2026-10-15T03:30:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:57.953+00:00 2026-10-15T03:30:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:58.957+00:00 2026-10-15T03:30:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:30:59.964+00:00 2026-10-15T03:30:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:30:59.964+00:00 2026-10-15T03:30:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:00.976+00:00 2026-10-15T03:31:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:00.976+00:00 2026-10-15T03:31:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:01.974+00:00 2026-10-15T03:31:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:01.975+00:00 2026-10-15T03:31:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:02.978+00:00 2026-10-15T03:31:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:02.978+00:00 2026-10-15T03:31:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:03.984+00:00 2026-10-15T03:31:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:03.984+00:00 2026-10-15T03:31:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:04.990+00:00 2026-10-15T03:31:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:04.990+00:00 2026-10-15T03:31:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:05.996+00:00 2026-10-15T03:31:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:05.996+00:00 2026-10-15T03:31:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:07.000+00:00 2026-10-15T03:31:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:07.000+00:00 2026-10-15T03:31:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:08.011+00:00 2026-10-15T03:31:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:08.011+00:00 2026-10-15T03:31:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:09.011+00:00 2026-10-15T03:31:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:09.011+00:00 2026-10-15T03:31:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:10.020+00:00 2026-10-15T03:31:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:10.020+00:00 2026-10-15T03:31:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:11.025+00:00 2026-10-15T03:31:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:11.025+00:00 2026-10-15T03:31:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:12.030+00:00 2026-10-15T03:31:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:12.030+00:00 2026-10-15T03:31:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:13.042+00:00 2026-10-15T03:31:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:13.043+00:00 2026-10-15T03:31:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:14.041+00:00 2026-10-15T03:31:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:14.041+00:00 2026-10-15T03:31:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:15.114+00:00 2026-10-15T03:31:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:15.114+00:00 2026-10-15T03:31:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:16.202+00:00 2026-10-15T03:31:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:16.203+00:00 2026-10-15T03:31:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:17.057+00:00 2026-10-15T03:31:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:17.057+00:00 2026-10-15T03:31:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:18.062+00:00 2026-10-15T03:31:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:18.063+00:00 2026-10-15T03:31:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:19.066+00:00 2026-10-15T03:31:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:19.067+00:00 2026-10-15T03:31:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:20.072+00:00 2026-10-15T03:31:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:20.073+00:00 2026-10-15T03:31:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:21.115+00:00 2026-10-15T03:31:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:21.115+00:00 2026-10-15T03:31:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:22.086+00:00 2026-10-15T03:31:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:22.086+00:00 2026-10-15T03:31:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:23.091+00:00 2026-10-15T03:31:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:23.091+00:00 2026-10-15T03:31:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:24.096+00:00 2026-10-15T03:31:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:24.096+00:00 2026-10-15T03:31:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:25.101+00:00 2026-10-15T03:31:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:25.101+00:00 2026-10-15T03:31:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:26.221+00:00 2026-10-15T03:31:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:26.221+00:00 2026-10-15T03:31:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:27.114+00:00 2026-10-15T03:31:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:27.114+00:00 2026-10-15T03:31:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:28.118+00:00 2026-10-15T03:31:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:28.118+00:00 2026-10-15T03:31:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:29.123+00:00 2026-10-15T03:31:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:29.123+00:00 2026-10-15T03:31:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:30.129+00:00 2026-10-15T03:31:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:30.129+00:00 2026-10-15T03:31:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:31.151+00:00 2026-10-15T03:31:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:31.151+00:00 2026-10-15T03:31:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:32.141+00:00 2026-10-15T03:31:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:32.141+00:00 2026-10-15T03:31:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:33.146+00:00 2026-10-15T03:31:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:33.146+00:00 2026-10-15T03:31:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:34.151+00:00 2026-10-15T03:31:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:34.151+00:00 2026-10-15T03:31:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:35.156+00:00 2026-10-15T03:31:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:35.157+00:00 2026-10-15T03:31:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:36.161+00:00 2026-10-15T03:31:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:36.161+00:00 2026-10-15T03:31:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:37.168+00:00 2026-10-15T03:31:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:37.168+00:00 2026-10-15T03:31:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:38.172+00:00 2026-10-15T03:31:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:38.172+00:00 2026-10-15T03:31:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:39.176+00:00 2026-10-15T03:31:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:39.176+00:00 2026-10-15T03:31:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:40.181+00:00 2026-10-15T03:31:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:40.182+00:00 2026-10-15T03:31:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:41.187+00:00 2026-10-15T03:31:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:41.187+00:00 2026-10-15T03:31:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:42.192+00:00 2026-10-15T03:31:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:42.192+00:00 2026-10-15T03:31:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:43.197+00:00 2026-10-15T03:31:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:43.197+00:00 2026-10-15T03:31:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:44.202+00:00 2026-10-15T03:31:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:44.203+00:00 2026-10-15T03:31:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:45.208+00:00 2026-10-15T03:31:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:45.208+00:00 2026-10-15T03:31:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:46.212+00:00 2026-10-15T03:31:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:46.212+00:00 2026-10-15T03:31:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:47.220+00:00 2026-10-15T03:31:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:47.220+00:00 2026-10-15T03:31:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:48.222+00:00 2026-10-15T03:31:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:48.223+00:00 2026-10-15T03:31:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:49.227+00:00 2026-10-15T03:31:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:49.227+00:00 2026-10-15T03:31:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:50.233+00:00 2026-10-15T03:31:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:50.233+00:00 2026-10-15T03:31:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:51.237+00:00 2026-10-15T03:31:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:51.238+00:00 2026-10-15T03:31:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:52.244+00:00 2026-10-15T03:31:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:52.244+00:00 2026-10-15T03:31:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:53.248+00:00 2026-10-15T03:31:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:53.248+00:00 2026-10-15T03:31:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:54.253+00:00 2026-10-15T03:31:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:54.253+00:00 2026-10-15T03:31:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:55.318+00:00 2026-10-15T03:31:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:55.319+00:00 2026-10-15T03:31:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:56.278+00:00 2026-10-15T03:31:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:56.278+00:00 2026-10-15T03:31:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:57.268+00:00 2026-10-15T03:31:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:57.268+00:00 2026-10-15T03:31:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:58.416+00:00 2026-10-15T03:31:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:58.416+00:00 2026-10-15T03:31:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:31:59.278+00:00 2026-10-15T03:31:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:31:59.278+00:00 2026-10-15T03:31:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:00.283+00:00 2026-10-15T03:32:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:00.283+00:00 2026-10-15T03:32:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:01.289+00:00 2026-10-15T03:32:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:01.291+00:00 2026-10-15T03:32:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:02.296+00:00 2026-10-15T03:32:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:02.296+00:00 2026-10-15T03:32:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:03.300+00:00 2026-10-15T03:32:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:03.300+00:00 2026-10-15T03:32:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:04.306+00:00 2026-10-15T03:32:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:04.306+00:00 2026-10-15T03:32:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:05.315+00:00 2026-10-15T03:32:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:05.315+00:00 2026-10-15T03:32:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:06.320+00:00 2026-10-15T03:32:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:06.320+00:00 2026-10-15T03:32:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:07.320+00:00 2026-10-15T03:32:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:07.320+00:00 2026-10-15T03:32:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:08.325+00:00 2026-10-15T03:32:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:08.325+00:00 2026-10-15T03:32:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:09.330+00:00 2026-10-15T03:32:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:09.330+00:00 2026-10-15T03:32:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:10.341+00:00 2026-10-15T03:32:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:10.341+00:00 2026-10-15T03:32:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:11.340+00:00 2026-10-15T03:32:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:11.340+00:00 2026-10-15T03:32:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:12.345+00:00 2026-10-15T03:32:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:12.345+00:00 2026-10-15T03:32:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:13.351+00:00 2026-10-15T03:32:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:13.351+00:00 2026-10-15T03:32:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:14.356+00:00 2026-10-15T03:32:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:14.356+00:00 2026-10-15T03:32:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:15.360+00:00 2026-10-15T03:32:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:15.361+00:00 2026-10-15T03:32:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:16.365+00:00 2026-10-15T03:32:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:16.365+00:00 2026-10-15T03:32:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:17.370+00:00 2026-10-15T03:32:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:17.370+00:00 2026-10-15T03:32:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:18.376+00:00 2026-10-15T03:32:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:18.376+00:00 2026-10-15T03:32:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:19.381+00:00 2026-10-15T03:32:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:19.381+00:00 2026-10-15T03:32:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:20.394+00:00 2026-10-15T03:32:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:20.394+00:00 2026-10-15T03:32:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:21.392+00:00 2026-10-15T03:32:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:21.392+00:00 2026-10-15T03:32:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:22.396+00:00 2026-10-15T03:32:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:22.396+00:00 2026-10-15T03:32:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:23.400+00:00 2026-10-15T03:32:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:23.400+00:00 2026-10-15T03:32:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:24.406+00:00 2026-10-15T03:32:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:24.406+00:00 2026-10-15T03:32:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:25.410+00:00 2026-10-15T03:32:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:25.411+00:00 2026-10-15T03:32:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:26.415+00:00 2026-10-15T03:32:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:26.415+00:00 2026-10-15T03:32:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:27.423+00:00 2026-10-15T03:32:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:27.423+00:00 2026-10-15T03:32:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:28.424+00:00 2026-10-15T03:32:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:28.424+00:00 2026-10-15T03:32:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:29.431+00:00 2026-10-15T03:32:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:29.431+00:00 2026-10-15T03:32:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:30.436+00:00 2026-10-15T03:32:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:30.436+00:00 2026-10-15T03:32:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:31.441+00:00 2026-10-15T03:32:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:31.441+00:00 2026-10-15T03:32:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:32.446+00:00 2026-10-15T03:32:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:32.446+00:00 2026-10-15T03:32:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:33.452+00:00 2026-10-15T03:32:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:33.452+00:00 2026-10-15T03:32:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:34.607+00:00 2026-10-15T03:32:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:34.607+00:00 2026-10-15T03:32:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:35.469+00:00 2026-10-15T03:32:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:35.469+00:00 2026-10-15T03:32:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:36.467+00:00 2026-10-15T03:32:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:36.467+00:00 2026-10-15T03:32:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:37.472+00:00 2026-10-15T03:32:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:37.472+00:00 2026-10-15T03:32:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:38.477+00:00 2026-10-15T03:32:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:38.477+00:00 2026-10-15T03:32:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:39.482+00:00 2026-10-15T03:32:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:39.483+00:00 2026-10-15T03:32:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:40.499+00:00 2026-10-15T03:32:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:40.499+00:00 2026-10-15T03:32:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:41.495+00:00 2026-10-15T03:32:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:41.495+00:00 2026-10-15T03:32:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:42.498+00:00 2026-10-15T03:32:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:42.498+00:00 2026-10-15T03:32:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:43.511+00:00 2026-10-15T03:32:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:43.511+00:00 2026-10-15T03:32:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:44.529+00:00 2026-10-15T03:32:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:44.529+00:00 2026-10-15T03:32:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:45.567+00:00 2026-10-15T03:32:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:45.567+00:00 2026-10-15T03:32:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:46.519+00:00 2026-10-15T03:32:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:46.520+00:00 2026-10-15T03:32:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:47.527+00:00 2026-10-15T03:32:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:47.528+00:00 2026-10-15T03:32:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:48.529+00:00 2026-10-15T03:32:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:48.529+00:00 2026-10-15T03:32:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:49.535+00:00 2026-10-15T03:32:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:49.535+00:00 2026-10-15T03:32:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:50.542+00:00 2026-10-15T03:32:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:50.543+00:00 2026-10-15T03:32:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:51.546+00:00 2026-10-15T03:32:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:51.546+00:00 2026-10-15T03:32:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:52.557+00:00 2026-10-15T03:32:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:52.557+00:00 2026-10-15T03:32:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:53.557+00:00 2026-10-15T03:32:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:53.557+00:00 2026-10-15T03:32:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:54.562+00:00 2026-10-15T03:32:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:54.562+00:00 2026-10-15T03:32:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:55.568+00:00 2026-10-15T03:32:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:55.568+00:00 2026-10-15T03:32:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:56.572+00:00 2026-10-15T03:32:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:56.572+00:00 2026-10-15T03:32:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:57.579+00:00 2026-10-15T03:32:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:57.579+00:00 2026-10-15T03:32:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:58.582+00:00 2026-10-15T03:32:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:58.582+00:00 2026-10-15T03:32:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:32:59.588+00:00 2026-10-15T03:32:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:32:59.588+00:00 2026-10-15T03:32:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:00.593+00:00 2026-10-15T03:33:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:00.593+00:00 2026-10-15T03:33:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:01.598+00:00 2026-10-15T03:33:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:01.599+00:00 2026-10-15T03:33:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:02.605+00:00 2026-10-15T03:33:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:02.605+00:00 2026-10-15T03:33:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:03.608+00:00 2026-10-15T03:33:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:03.608+00:00 2026-10-15T03:33:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:04.614+00:00 2026-10-15T03:33:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:04.614+00:00 2026-10-15T03:33:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:05.620+00:00 2026-10-15T03:33:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:05.620+00:00 2026-10-15T03:33:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:06.624+00:00 2026-10-15T03:33:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:06.624+00:00 2026-10-15T03:33:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:07.629+00:00 2026-10-15T03:33:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:07.629+00:00 2026-10-15T03:33:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:08.635+00:00 2026-10-15T03:33:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:08.635+00:00 2026-10-15T03:33:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:09.640+00:00 2026-10-15T03:33:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:09.640+00:00 2026-10-15T03:33:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:10.646+00:00 2026-10-15T03:33:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:10.646+00:00 2026-10-15T03:33:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:11.651+00:00 2026-10-15T03:33:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:11.652+00:00 2026-10-15T03:33:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:12.663+00:00 2026-10-15T03:33:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:12.664+00:00 2026-10-15T03:33:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:13.667+00:00 2026-10-15T03:33:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:13.667+00:00 2026-10-15T03:33:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:14.667+00:00 2026-10-15T03:33:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:14.667+00:00 2026-10-15T03:33:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:15.671+00:00 2026-10-15T03:33:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:15.671+00:00 2026-10-15T03:33:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:16.676+00:00 2026-10-15T03:33:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:16.676+00:00 2026-10-15T03:33:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:17.681+00:00 2026-10-15T03:33:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:17.681+00:00 2026-10-15T03:33:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:18.686+00:00 2026-10-15T03:33:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:18.686+00:00 2026-10-15T03:33:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:19.695+00:00 2026-10-15T03:33:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:19.695+00:00 2026-10-15T03:33:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:20.696+00:00 2026-10-15T03:33:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:20.696+00:00 2026-10-15T03:33:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:21.715+00:00 2026-10-15T03:33:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:21.715+00:00 2026-10-15T03:33:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:22.706+00:00 2026-10-15T03:33:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:22.706+00:00 2026-10-15T03:33:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:23.711+00:00 2026-10-15T03:33:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:23.711+00:00 2026-10-15T03:33:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:24.717+00:00 2026-10-15T03:33:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:24.717+00:00 2026-10-15T03:33:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:25.721+00:00 2026-10-15T03:33:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:25.721+00:00 2026-10-15T03:33:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:26.728+00:00 2026-10-15T03:33:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:26.728+00:00 2026-10-15T03:33:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:27.733+00:00 2026-10-15T03:33:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:27.733+00:00 2026-10-15T03:33:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:28.737+00:00 2026-10-15T03:33:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:28.737+00:00 2026-10-15T03:33:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:29.936+00:00 2026-10-15T03:33:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:29.937+00:00 2026-10-15T03:33:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:30.747+00:00 2026-10-15T03:33:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:30.747+00:00 2026-10-15T03:33:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:31.752+00:00 2026-10-15T03:33:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:31.752+00:00 2026-10-15T03:33:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:32.759+00:00 2026-10-15T03:33:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:32.760+00:00 2026-10-15T03:33:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:33.772+00:00 2026-10-15T03:33:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:33.772+00:00 2026-10-15T03:33:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:34.769+00:00 2026-10-15T03:33:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:34.769+00:00 2026-10-15T03:33:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:35.776+00:00 2026-10-15T03:33:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:35.776+00:00 2026-10-15T03:33:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:36.779+00:00 2026-10-15T03:33:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:36.779+00:00 2026-10-15T03:33:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:37.784+00:00 2026-10-15T03:33:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:37.784+00:00 2026-10-15T03:33:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:38.789+00:00 2026-10-15T03:33:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:38.790+00:00 2026-10-15T03:33:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:39.795+00:00 2026-10-15T03:33:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:39.795+00:00 2026-10-15T03:33:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:40.799+00:00 2026-10-15T03:33:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:40.799+00:00 2026-10-15T03:33:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:41.806+00:00 2026-10-15T03:33:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:41.806+00:00 2026-10-15T03:33:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:42.809+00:00 2026-10-15T03:33:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:42.809+00:00 2026-10-15T03:33:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:43.815+00:00 2026-10-15T03:33:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:43.815+00:00 2026-10-15T03:33:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:44.887+00:00 2026-10-15T03:33:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:44.887+00:00 2026-10-15T03:33:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:45.826+00:00 2026-10-15T03:33:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:45.826+00:00 2026-10-15T03:33:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:46.831+00:00 2026-10-15T03:33:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:46.831+00:00 2026-10-15T03:33:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:47.836+00:00 2026-10-15T03:33:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:47.836+00:00 2026-10-15T03:33:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:48.841+00:00 2026-10-15T03:33:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:48.841+00:00 2026-10-15T03:33:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:49.847+00:00 2026-10-15T03:33:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:49.847+00:00 2026-10-15T03:33:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:50.852+00:00 2026-10-15T03:33:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:50.852+00:00 2026-10-15T03:33:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:52.084+00:00 2026-10-15T03:33:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:52.085+00:00 2026-10-15T03:33:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:52.866+00:00 2026-10-15T03:33:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:52.866+00:00 2026-10-15T03:33:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:53.867+00:00 2026-10-15T03:33:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:53.867+00:00 2026-10-15T03:33:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:54.874+00:00 2026-10-15T03:33:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:54.874+00:00 2026-10-15T03:33:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:55.880+00:00 2026-10-15T03:33:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:55.880+00:00 2026-10-15T03:33:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:56.885+00:00 2026-10-15T03:33:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:56.885+00:00 2026-10-15T03:33:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:57.889+00:00 2026-10-15T03:33:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:57.889+00:00 2026-10-15T03:33:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:58.895+00:00 2026-10-15T03:33:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:58.895+00:00 2026-10-15T03:33:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:33:59.899+00:00 2026-10-15T03:33:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:33:59.899+00:00 2026-10-15T03:33:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:00.904+00:00 2026-10-15T03:34:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:00.904+00:00 2026-10-15T03:34:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:02.024+00:00 2026-10-15T03:34:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:02.024+00:00 2026-10-15T03:34:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:02.918+00:00 2026-10-15T03:34:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:02.918+00:00 2026-10-15T03:34:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:03.919+00:00 2026-10-15T03:34:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:03.920+00:00 2026-10-15T03:34:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:04.926+00:00 2026-10-15T03:34:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:04.926+00:00 2026-10-15T03:34:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:05.931+00:00 2026-10-15T03:34:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:05.931+00:00 2026-10-15T03:34:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:06.935+00:00 2026-10-15T03:34:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:06.936+00:00 2026-10-15T03:34:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:07.942+00:00 2026-10-15T03:34:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:07.942+00:00 2026-10-15T03:34:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:08.945+00:00 2026-10-15T03:34:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:08.946+00:00 2026-10-15T03:34:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:09.950+00:00 2026-10-15T03:34:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:09.951+00:00 2026-10-15T03:34:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:10.958+00:00 2026-10-15T03:34:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:10.958+00:00 2026-10-15T03:34:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:11.966+00:00 2026-10-15T03:34:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:11.966+00:00 2026-10-15T03:34:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:12.966+00:00 2026-10-15T03:34:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:12.966+00:00 2026-10-15T03:34:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:13.973+00:00 2026-10-15T03:34:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:13.973+00:00 2026-10-15T03:34:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:14.977+00:00 2026-10-15T03:34:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:14.977+00:00 2026-10-15T03:34:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:15.981+00:00 2026-10-15T03:34:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:15.981+00:00 2026-10-15T03:34:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:16.986+00:00 2026-10-15T03:34:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:16.986+00:00 2026-10-15T03:34:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:17.992+00:00 2026-10-15T03:34:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:17.992+00:00 2026-10-15T03:34:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:18.997+00:00 2026-10-15T03:34:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:18.998+00:00 2026-10-15T03:34:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:20.002+00:00 2026-10-15T03:34:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:20.002+00:00 2026-10-15T03:34:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:21.007+00:00 2026-10-15T03:34:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:21.007+00:00 2026-10-15T03:34:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:22.019+00:00 2026-10-15T03:34:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:22.019+00:00 2026-10-15T03:34:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:23.018+00:00 2026-10-15T03:34:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:23.018+00:00 2026-10-15T03:34:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:24.031+00:00 2026-10-15T03:34:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:24.031+00:00 2026-10-15T03:34:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:25.030+00:00 2026-10-15T03:34:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:25.030+00:00 2026-10-15T03:34:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:26.053+00:00 2026-10-15T03:34:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:26.053+00:00 2026-10-15T03:34:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:27.043+00:00 2026-10-15T03:34:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:27.043+00:00 2026-10-15T03:34:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:28.043+00:00 2026-10-15T03:34:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:28.043+00:00 2026-10-15T03:34:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:29.048+00:00 2026-10-15T03:34:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:29.048+00:00 2026-10-15T03:34:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:30.053+00:00 2026-10-15T03:34:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:30.053+00:00 2026-10-15T03:34:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:31.172+00:00 2026-10-15T03:34:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:31.172+00:00 2026-10-15T03:34:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:32.063+00:00 2026-10-15T03:34:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:32.064+00:00 2026-10-15T03:34:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:33.070+00:00 2026-10-15T03:34:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:33.070+00:00 2026-10-15T03:34:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:34.074+00:00 2026-10-15T03:34:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:34.074+00:00 2026-10-15T03:34:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:35.079+00:00 2026-10-15T03:34:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:35.079+00:00 2026-10-15T03:34:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:36.170+00:00 2026-10-15T03:34:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:36.170+00:00 2026-10-15T03:34:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:37.090+00:00 2026-10-15T03:34:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:37.090+00:00 2026-10-15T03:34:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:38.095+00:00 2026-10-15T03:34:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:38.095+00:00 2026-10-15T03:34:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:39.159+00:00 2026-10-15T03:34:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:39.159+00:00 2026-10-15T03:34:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:40.105+00:00 2026-10-15T03:34:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:40.105+00:00 2026-10-15T03:34:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:41.110+00:00 2026-10-15T03:34:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:41.111+00:00 2026-10-15T03:34:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:42.115+00:00 2026-10-15T03:34:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:42.115+00:00 2026-10-15T03:34:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:43.120+00:00 2026-10-15T03:34:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:43.120+00:00 2026-10-15T03:34:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:44.126+00:00 2026-10-15T03:34:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:44.126+00:00 2026-10-15T03:34:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:45.130+00:00 2026-10-15T03:34:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:45.131+00:00 2026-10-15T03:34:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:46.135+00:00 2026-10-15T03:34:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:46.135+00:00 2026-10-15T03:34:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:47.141+00:00 2026-10-15T03:34:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:47.141+00:00 2026-10-15T03:34:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:48.146+00:00 2026-10-15T03:34:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:48.147+00:00 2026-10-15T03:34:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:49.154+00:00 2026-10-15T03:34:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:49.154+00:00 2026-10-15T03:34:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:50.156+00:00 2026-10-15T03:34:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:50.156+00:00 2026-10-15T03:34:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:51.162+00:00 2026-10-15T03:34:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:51.162+00:00 2026-10-15T03:34:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:52.168+00:00 2026-10-15T03:34:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:52.168+00:00 2026-10-15T03:34:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:53.178+00:00 2026-10-15T03:34:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:53.179+00:00 2026-10-15T03:34:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:54.178+00:00 2026-10-15T03:34:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:54.178+00:00 2026-10-15T03:34:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:55.182+00:00 2026-10-15T03:34:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:55.183+00:00 2026-10-15T03:34:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:56.188+00:00 2026-10-15T03:34:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:56.188+00:00 2026-10-15T03:34:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:57.193+00:00 2026-10-15T03:34:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:57.193+00:00 2026-10-15T03:34:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:58.197+00:00 2026-10-15T03:34:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:58.197+00:00 2026-10-15T03:34:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:34:59.202+00:00 2026-10-15T03:34:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:34:59.203+00:00 2026-10-15T03:34:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:00.208+00:00 2026-10-15T03:35:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:00.208+00:00 2026-10-15T03:35:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:01.212+00:00 2026-10-15T03:35:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:01.213+00:00 2026-10-15T03:35:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:02.218+00:00 2026-10-15T03:35:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:02.218+00:00 2026-10-15T03:35:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:03.222+00:00 2026-10-15T03:35:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:03.222+00:00 2026-10-15T03:35:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:04.227+00:00 2026-10-15T03:35:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:04.228+00:00 2026-10-15T03:35:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:05.232+00:00 2026-10-15T03:35:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:05.233+00:00 2026-10-15T03:35:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:06.238+00:00 2026-10-15T03:35:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:06.238+00:00 2026-10-15T03:35:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:07.243+00:00 2026-10-15T03:35:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:07.243+00:00 2026-10-15T03:35:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:08.248+00:00 2026-10-15T03:35:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:08.248+00:00 2026-10-15T03:35:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:09.253+00:00 2026-10-15T03:35:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:09.253+00:00 2026-10-15T03:35:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:10.259+00:00 2026-10-15T03:35:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:10.259+00:00 2026-10-15T03:35:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:11.268+00:00 2026-10-15T03:35:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:11.268+00:00 2026-10-15T03:35:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:12.269+00:00 2026-10-15T03:35:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:12.269+00:00 2026-10-15T03:35:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:13.277+00:00 2026-10-15T03:35:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:13.277+00:00 2026-10-15T03:35:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:14.281+00:00 2026-10-15T03:35:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:14.281+00:00 2026-10-15T03:35:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:15.287+00:00 2026-10-15T03:35:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:15.287+00:00 2026-10-15T03:35:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:16.433+00:00 2026-10-15T03:35:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:16.433+00:00 2026-10-15T03:35:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:17.298+00:00 2026-10-15T03:35:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:17.298+00:00 2026-10-15T03:35:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:18.302+00:00 2026-10-15T03:35:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:18.302+00:00 2026-10-15T03:35:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:19.313+00:00 2026-10-15T03:35:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:19.314+00:00 2026-10-15T03:35:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:20.311+00:00 2026-10-15T03:35:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:20.311+00:00 2026-10-15T03:35:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:21.319+00:00 2026-10-15T03:35:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:21.319+00:00 2026-10-15T03:35:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:22.322+00:00 2026-10-15T03:35:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:22.323+00:00 2026-10-15T03:35:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:23.329+00:00 2026-10-15T03:35:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:23.329+00:00 2026-10-15T03:35:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:24.333+00:00 2026-10-15T03:35:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:24.333+00:00 2026-10-15T03:35:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:25.338+00:00 2026-10-15T03:35:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:25.338+00:00 2026-10-15T03:35:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:26.343+00:00 2026-10-15T03:35:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:26.343+00:00 2026-10-15T03:35:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:27.348+00:00 2026-10-15T03:35:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:27.348+00:00 2026-10-15T03:35:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:28.353+00:00 2026-10-15T03:35:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:28.353+00:00 2026-10-15T03:35:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:29.359+00:00 2026-10-15T03:35:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:29.359+00:00 2026-10-15T03:35:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:30.367+00:00 2026-10-15T03:35:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:30.367+00:00 2026-10-15T03:35:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:31.368+00:00 2026-10-15T03:35:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:31.368+00:00 2026-10-15T03:35:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:32.373+00:00 2026-10-15T03:35:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:32.373+00:00 2026-10-15T03:35:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:33.378+00:00 2026-10-15T03:35:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:33.378+00:00 2026-10-15T03:35:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:34.383+00:00 2026-10-15T03:35:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:34.384+00:00 2026-10-15T03:35:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:35.389+00:00 2026-10-15T03:35:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:35.389+00:00 2026-10-15T03:35:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:36.394+00:00 2026-10-15T03:35:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:36.394+00:00 2026-10-15T03:35:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:37.399+00:00 2026-10-15T03:35:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:37.399+00:00 2026-10-15T03:35:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:38.404+00:00 2026-10-15T03:35:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:38.404+00:00 2026-10-15T03:35:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:39.408+00:00 2026-10-15T03:35:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:39.408+00:00 2026-10-15T03:35:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:40.416+00:00 2026-10-15T03:35:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:40.417+00:00 2026-10-15T03:35:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:41.424+00:00 2026-10-15T03:35:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:41.424+00:00 2026-10-15T03:35:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:42.423+00:00 2026-10-15T03:35:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:42.423+00:00 2026-10-15T03:35:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:43.428+00:00 2026-10-15T03:35:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:43.428+00:00 2026-10-15T03:35:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:44.433+00:00 2026-10-15T03:35:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:44.433+00:00 2026-10-15T03:35:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:45.439+00:00 2026-10-15T03:35:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:45.439+00:00 2026-10-15T03:35:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:46.443+00:00 2026-10-15T03:35:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:46.443+00:00 2026-10-15T03:35:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:47.457+00:00 2026-10-15T03:35:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:47.457+00:00 2026-10-15T03:35:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:48.453+00:00 2026-10-15T03:35:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:48.453+00:00 2026-10-15T03:35:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:49.461+00:00 2026-10-15T03:35:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:49.461+00:00 2026-10-15T03:35:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:50.464+00:00 2026-10-15T03:35:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:50.464+00:00 2026-10-15T03:35:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:51.471+00:00 2026-10-15T03:35:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:51.471+00:00 2026-10-15T03:35:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:52.477+00:00 2026-10-15T03:35:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:52.477+00:00 2026-10-15T03:35:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:53.480+00:00 2026-10-15T03:35:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:53.480+00:00 2026-10-15T03:35:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:54.484+00:00 2026-10-15T03:35:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:54.484+00:00 2026-10-15T03:35:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:55.490+00:00 2026-10-15T03:35:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:55.490+00:00 2026-10-15T03:35:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:56.494+00:00 2026-10-15T03:35:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:56.495+00:00 2026-10-15T03:35:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:57.500+00:00 2026-10-15T03:35:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:57.500+00:00 2026-10-15T03:35:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:58.506+00:00 2026-10-15T03:35:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:58.506+00:00 2026-10-15T03:35:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:35:59.520+00:00 2026-10-15T03:35:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:35:59.520+00:00 2026-10-15T03:35:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:00.520+00:00 2026-10-15T03:36:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:00.520+00:00 2026-10-15T03:36:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:01.522+00:00 2026-10-15T03:36:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:01.522+00:00 2026-10-15T03:36:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:02.527+00:00 2026-10-15T03:36:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:02.527+00:00 2026-10-15T03:36:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:03.533+00:00 2026-10-15T03:36:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:03.533+00:00 2026-10-15T03:36:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:04.544+00:00 2026-10-15T03:36:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:04.544+00:00 2026-10-15T03:36:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:05.544+00:00 2026-10-15T03:36:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:05.544+00:00 2026-10-15T03:36:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:06.550+00:00 2026-10-15T03:36:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:06.550+00:00 2026-10-15T03:36:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:07.556+00:00 2026-10-15T03:36:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:07.556+00:00 2026-10-15T03:36:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:08.559+00:00 2026-10-15T03:36:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:08.559+00:00 2026-10-15T03:36:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:09.574+00:00 2026-10-15T03:36:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:09.575+00:00 2026-10-15T03:36:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:10.570+00:00 2026-10-15T03:36:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:10.570+00:00 2026-10-15T03:36:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:11.576+00:00 2026-10-15T03:36:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:11.576+00:00 2026-10-15T03:36:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:12.581+00:00 2026-10-15T03:36:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:12.581+00:00 2026-10-15T03:36:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:13.588+00:00 2026-10-15T03:36:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:13.588+00:00 2026-10-15T03:36:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:14.592+00:00 2026-10-15T03:36:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:14.593+00:00 2026-10-15T03:36:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:15.597+00:00 2026-10-15T03:36:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:15.598+00:00 2026-10-15T03:36:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:16.602+00:00 2026-10-15T03:36:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:16.602+00:00 2026-10-15T03:36:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:17.608+00:00 2026-10-15T03:36:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:17.608+00:00 2026-10-15T03:36:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:18.612+00:00 2026-10-15T03:36:18Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:18.612+00:00 2026-10-15T03:36:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:18.612+00:00 2026-10-15T03:36:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:19.617+00:00 2026-10-15T03:36:19Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:19.617+00:00 2026-10-15T03:36:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:19.617+00:00 2026-10-15T03:36:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:20.622+00:00 2026-10-15T03:36:20Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:20.622+00:00 2026-10-15T03:36:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:20.622+00:00 2026-10-15T03:36:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:21.627+00:00 2026-10-15T03:36:21Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:21.627+00:00 2026-10-15T03:36:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:21.627+00:00 2026-10-15T03:36:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:22.631+00:00 2026-10-15T03:36:22Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:22.631+00:00 2026-10-15T03:36:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:22.631+00:00 2026-10-15T03:36:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:23.637+00:00 2026-10-15T03:36:23Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:23.637+00:00 2026-10-15T03:36:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:23.637+00:00 2026-10-15T03:36:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:24.665+00:00 2026-10-15T03:36:24Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:24.665+00:00 2026-10-15T03:36:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:24.665+00:00 2026-10-15T03:36:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:25.648+00:00 2026-10-15T03:36:25Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:25.648+00:00 2026-10-15T03:36:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:25.648+00:00 2026-10-15T03:36:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:26.652+00:00 2026-10-15T03:36:26Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:26.652+00:00 2026-10-15T03:36:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:26.652+00:00 2026-10-15T03:36:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:27.657+00:00 2026-10-15T03:36:27Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:27.657+00:00 2026-10-15T03:36:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:27.657+00:00 2026-10-15T03:36:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:28.663+00:00 2026-10-15T03:36:28Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:28.663+00:00 2026-10-15T03:36:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:28.663+00:00 2026-10-15T03:36:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:29.668+00:00 2026-10-15T03:36:29Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:29.668+00:00 2026-10-15T03:36:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:29.668+00:00 2026-10-15T03:36:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:30.672+00:00 2026-10-15T03:36:30Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:30.672+00:00 2026-10-15T03:36:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:30.672+00:00 2026-10-15T03:36:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:31.676+00:00 2026-10-15T03:36:31Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:31.676+00:00 2026-10-15T03:36:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:31.676+00:00 2026-10-15T03:36:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:32.682+00:00 2026-10-15T03:36:32Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:32.682+00:00 2026-10-15T03:36:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:32.682+00:00 2026-10-15T03:36:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:33.688+00:00 2026-10-15T03:36:33Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:33.688+00:00 2026-10-15T03:36:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:33.688+00:00 2026-10-15T03:36:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:34.702+00:00 2026-10-15T03:36:34Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:34.702+00:00 2026-10-15T03:36:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:34.702+00:00 2026-10-15T03:36:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:35.698+00:00 2026-10-15T03:36:35Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:35.698+00:00 2026-10-15T03:36:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:35.698+00:00 2026-10-15T03:36:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:36.704+00:00 2026-10-15T03:36:36Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:36.704+00:00 2026-10-15T03:36:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:36.704+00:00 2026-10-15T03:36:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:37.709+00:00 2026-10-15T03:36:37Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:37.710+00:00 2026-10-15T03:36:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:37.710+00:00 2026-10-15T03:36:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:38.714+00:00 2026-10-15T03:36:38Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:38.714+00:00 2026-10-15T03:36:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:38.715+00:00 2026-10-15T03:36:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:39.719+00:00 2026-10-15T03:36:39Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:39.719+00:00 2026-10-15T03:36:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:39.719+00:00 2026-10-15T03:36:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:40.724+00:00 2026-10-15T03:36:40Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:40.724+00:00 2026-10-15T03:36:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:40.724+00:00 2026-10-15T03:36:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:41.729+00:00 2026-10-15T03:36:41Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:41.729+00:00 2026-10-15T03:36:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:41.729+00:00 2026-10-15T03:36:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:42.734+00:00 2026-10-15T03:36:42Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:42.734+00:00 2026-10-15T03:36:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:42.734+00:00 2026-10-15T03:36:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:43.740+00:00 2026-10-15T03:36:43Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:43.740+00:00 2026-10-15T03:36:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:43.740+00:00 2026-10-15T03:36:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:44.744+00:00 2026-10-15T03:36:44Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:44.744+00:00 2026-10-15T03:36:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:44.744+00:00 2026-10-15T03:36:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:45.749+00:00 2026-10-15T03:36:45Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:45.749+00:00 2026-10-15T03:36:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:45.749+00:00 2026-10-15T03:36:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:46.753+00:00 2026-10-15T03:36:46Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:46.753+00:00 2026-10-15T03:36:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:46.753+00:00 2026-10-15T03:36:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:47.758+00:00 2026-10-15T03:36:47Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:47.758+00:00 2026-10-15T03:36:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:47.758+00:00 2026-10-15T03:36:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:48.771+00:00 2026-10-15T03:36:48Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:48.772+00:00 2026-10-15T03:36:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:48.772+00:00 2026-10-15T03:36:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:36:49.770+00:00 2026-10-15T03:36:49Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T03:36:49.770+00:00 2026-10-15T03:36:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:36:49.770+00:00 2026-10-15T03:36:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt deleted file mode 100644 index 6070ba6b..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample1.txt +++ /dev/null @@ -1,2 +0,0 @@ - bytes: 16171806671 0 0 0 16171746116 0 3948316 0 - READ: 18362 18362 0 2864472 16174096460 285 180396 181025 0 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt deleted file mode 100644 index 0f0d837e..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/sampler-mountstats-sample2.txt +++ /dev/null @@ -1,2 +0,0 @@ - bytes: 16393056335 0 0 0 16392991684 0 4002204 0 - READ: 18796 18796 0 2932176 16395397580 292 204482 205130 0 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log b/runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log deleted file mode 100644 index bd7aeca5..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T03:14:14.700+00:00 2026-10-15T03:14:14.660958634Z {"level":"info","ts":"2026-10-15T03:14:14Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"73e6ab21-a0b9-4211-8b1e-8d228cf38838","claim":"claim-lora-qwen-qwen3-8b-0-trainer-013a69b1","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} -2026-10-15T03:31:10.923+00:00 2026-10-15T03:31:10.911470512Z {"level":"info","ts":"2026-10-15T03:31:10Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"2b07d6ec-2b25-4dba-a67d-607db38e8fea","claim":"claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json b/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json deleted file mode 100644 index c474d169..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/timeline.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "case": "C-8b-h100-cold", - "collected": "2026-10-15T03:38:49.573+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 03:14:13.682000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen3-8B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 03:14:13.683000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 03:31:02.295000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "128ed0fe-59ba-43e7-b0f2-7997322d8233" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 03:31:05.464000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 03:31:08.365000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 03:31:10.461000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 03:31:10.462000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 03:31:12.490000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 03:31:12.490000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 03:36:43.884000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4\n\nQuestion: What is " - }, - "client.end": { - "ts": "2026-10-15 03:36:43.886000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen3-8b-0-sampler", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "pod": "orw-lora-qwen-qwen3-8b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 03:31:10+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 03:31:10.295000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 03:31:11.907000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 03:31:11.907000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 03:32:51+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 03:31:11+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 03:31:11+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 03:32:51+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 03:32:51+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 03:31:11+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 03:32:50+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 03:31:11+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "sampler.event_Pulling": { - "ts": "2026-10-15 03:31:15+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 03:32:50+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m35.306s (1m35.306s i", - "kubelet_pull_s": 95.306, - "pulled_from_cache": false - }, - "sampler.event_Created": { - "ts": "2026-10-15 03:32:50+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 03:32:50+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 03:32:57.768739+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 03:33:16.117730+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:33:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 03:33:17.209275+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:33:17 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 03:33:18.247405+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:33:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 03:35:00.792211+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:00 [default_loader.py:430] Loading weights took 102.55 seconds", - "self_reported_s": 102.55 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 03:35:01.444736+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:01 [model_runner.py:428] Model loading took 18.17 GiB memory and 104.493682 second", - "self_reported_s": 104.493682 - }, - "sampler.compile_done": { - "ts": "2026-10-15 03:35:37.706264+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:37 [monitor.py:53] torch.compile took 35.25 s in total", - "self_reported_s": 35.25 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 03:35:57.941422+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:57 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", - "self_reported_s": 16.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 03:35:58.562053+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:35:58 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 03:36:17.004554+00:00", - "source": "container log: (EngineCore pid=168) INFO 10-15 03:36:17 [core.py:372] init engine (profile, create kv cache, warmup model) took 75.55 s", - "self_reported_s": 75.55 - }, - "sampler.ready_key": { - "ts": "2026-10-15 03:36:18+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" - }, - "trainer": { - "workload": "lora-qwen-qwen3-8b-0-trainer", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "pod": "orw-lora-qwen-qwen3-8b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 03:14:14+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 03:14:14.472000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 03:14:14.472000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 03:14:14.472000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 03:15:52+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 03:14:14+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 03:14:14+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 03:15:52+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 03:15:52+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 03:14:14+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 03:15:51+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 03:14:14+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "trainer.event_Pulling": { - "ts": "2026-10-15 03:14:19+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 03:15:51+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m32.497s (1m32.497s i", - "kubelet_pull_s": 92.497, - "pulled_from_cache": false - }, - "trainer.event_Created": { - "ts": "2026-10-15 03:15:51+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 03:15:51+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 03:15:56.012903+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 03:15:56.267417+00:00", - "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 03:30:59.760141+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 03:30:59.761631+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 03:30:59+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt deleted file mode 100644 index 433537d3..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample1.txt +++ /dev/null @@ -1,14 +0,0 @@ -device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 - opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none - age: 403 - events: 58 1574 0 0 44 61 1744 0 0 12462 0 0 32 0 28 0 0 23 0 0 0 0 0 0 0 0 0 - bytes: 11495181 0 0 0 3265001325 0 797317 0 - READ: 12459 12459 0 1943604 3266596084 221 584552 585076 0 -device /dev/md0 mounted on /etc/hosts with fstype ext4 -device /dev/md0 mounted on /dev/termination-log with fstype ext4 -device /dev/md0 mounted on /etc/hostname with fstype ext4 -device /dev/md0 mounted on /etc/resolv.conf with fstype ext4 -device shm mounted on /dev/shm with fstype tmpfs -device /dev/nvme0n1p1 mounted on /usr/bin/nvidia-cuda-mps-control with fstype ext4 -device /dev/nvme0n1p1 mounted on /usr/bin/nvidia-cuda-mps-server with fstype ext4 -device /dev/nvme0n1p1 mounted on /usr/bin/nvidia-debugdump with fstype ext4 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt deleted file mode 100644 index b84369a0..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/trainer-mountstats-sample2.txt +++ /dev/null @@ -1,2 +0,0 @@ - bytes: 11495181 0 0 0 3943458669 0 962892 0 - READ: 15070 15070 0 2350920 3945387636 265 704690 705322 0 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt deleted file mode 100644 index c83d56c7..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T03:14:09.568+00:00 diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt b/runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt deleted file mode 100644 index 0d162c07..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json b/runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json deleted file mode 100644 index 11c30084..00000000 --- a/runs/2026-10-15_cold_start/C-8b-h100-cold/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T03:31:10Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035171447887008", - "uid": "4637c9e3-a819-432d-9edb-58a1522564c4" - }, - "spec": { - "accelerator": { - "memory": "29Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "name": "worker", - "resources": { - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "464502aa-adbd-4913-8de9-9cf72fa2e24d", - "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T03:32:51Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-4637c9e3", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "29Gi", - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T03:14:14Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792034152699023004", - "uid": "013a69b1-c398-4e81-a5c5-90edf5da837a" - }, - "spec": { - "accelerator": { - "memory": "20Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "name": "worker", - "resources": { - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "bbe9937b-9591-4fe5-9cd5-1845592a917d", - "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T03:15:52Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-013a69b1", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "20Gi", - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt deleted file mode 100644 index 5d1863d9..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/NOTE.txt +++ /dev/null @@ -1 +0,0 @@ -interrupted: port-forward outage ~03:43-04:23 UTC; reaper released workers 03:45:24; trainer load 03:39:13->03:43:39 (266 s) is valid diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log deleted file mode 100644 index 246ca3b1..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/api-server.log +++ /dev/null @@ -1,38 +0,0 @@ -2026-10-15T03:39:03.941+00:00 2026-10-15T03:39:03.931120235Z INFO: 127.0.0.1:42054 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T03:39:03.975+00:00 2026-10-15T03:39:03.964551019Z INFO: 10.32.3.1:38928 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:39:04.179+00:00 2026-10-15T03:39:04.169671310Z INFO: 127.0.0.1:42060 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T03:39:04.245+00:00 2026-10-15T03:39:04.236451920Z INFO: 127.0.0.1:42060 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T03:39:04.488+00:00 2026-10-15T03:39:04.478554501Z [API_SERVER] enqueued op=create_model request_id=ddb4615c-4bb0-44b0-92a1-c79e668c7324 model_id=ddb4615c-4bb0-44b0-92a1-c79e668c7324 active_set=Qwen/Qwen3-8B-1 -2026-10-15T03:39:04.489+00:00 2026-10-15T03:39:04.479576649Z INFO: 127.0.0.1:42062 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T03:39:13.977+00:00 2026-10-15T03:39:13.964648602Z INFO: 10.32.3.1:37426 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:39:14.434+00:00 2026-10-15T03:39:14.425265125Z INFO: 127.0.0.1:55092 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:39:23.976+00:00 2026-10-15T03:39:23.964483470Z INFO: 10.32.3.1:37262 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:39:33.977+00:00 2026-10-15T03:39:33.964871846Z INFO: 10.32.3.1:58992 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:39:43.975+00:00 2026-10-15T03:39:43.964576360Z INFO: 10.32.3.1:52666 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:39:53.976+00:00 2026-10-15T03:39:53.964501809Z INFO: 10.32.3.1:52688 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:39:54.635+00:00 2026-10-15T03:39:54.609937412Z INFO: 127.0.0.1:51376 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:40:04.035+00:00 2026-10-15T03:40:03.964991692Z INFO: 10.32.3.1:41216 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:40:13.975+00:00 2026-10-15T03:40:13.963993568Z INFO: 10.32.3.1:53146 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:40:23.977+00:00 2026-10-15T03:40:23.965037340Z INFO: 10.32.3.1:40128 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:40:33.977+00:00 2026-10-15T03:40:33.965609459Z INFO: 10.32.3.1:54704 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:40:43.976+00:00 2026-10-15T03:40:43.965003505Z INFO: 10.32.3.1:40814 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:40:44.810+00:00 2026-10-15T03:40:44.800601117Z INFO: 127.0.0.1:59960 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:40:53.976+00:00 2026-10-15T03:40:53.964698725Z INFO: 10.32.3.1:36634 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:41:03.977+00:00 2026-10-15T03:41:03.964982471Z INFO: 10.32.3.1:54336 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:41:13.977+00:00 2026-10-15T03:41:13.964439663Z INFO: 10.32.3.1:57996 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:41:23.977+00:00 2026-10-15T03:41:23.964617258Z INFO: 10.32.3.1:33020 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:41:25.003+00:00 2026-10-15T03:41:24.994100729Z INFO: 127.0.0.1:60116 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:41:33.977+00:00 2026-10-15T03:41:33.964563446Z INFO: 10.32.3.1:39176 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:41:43.978+00:00 2026-10-15T03:41:43.965156711Z INFO: 10.32.3.1:43750 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:41:53.982+00:00 2026-10-15T03:41:53.964453599Z INFO: 10.32.3.1:43488 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:42:03.976+00:00 2026-10-15T03:42:03.965028296Z INFO: 10.32.3.1:56500 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:42:13.976+00:00 2026-10-15T03:42:13.964101051Z INFO: 10.32.3.1:32798 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:42:15.186+00:00 2026-10-15T03:42:15.177104960Z INFO: 127.0.0.1:42622 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:42:23.976+00:00 2026-10-15T03:42:23.964020928Z INFO: 10.32.3.1:56208 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:42:33.976+00:00 2026-10-15T03:42:33.964949522Z INFO: 10.32.3.1:49210 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:42:43.975+00:00 2026-10-15T03:42:43.964068045Z INFO: 10.32.3.1:54612 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:42:53.977+00:00 2026-10-15T03:42:53.964413969Z INFO: 10.32.3.1:43698 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:43:03.976+00:00 2026-10-15T03:43:03.964619748Z INFO: 10.32.3.1:44292 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:43:05.369+00:00 2026-10-15T03:43:05.359540334Z INFO: 127.0.0.1:46534 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T03:43:13.978+00:00 2026-10-15T03:43:13.964574300Z INFO: 10.32.3.1:53490 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T03:48:57.227+00:00 error: http2: client connection lost diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl deleted file mode 100644 index 501cd45a..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"ts": "2026-10-15T03:39:03.686+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T03:39:03.686+00:00", "t": 0.001, "phase": "create_model.start"} -{"ts": "2026-10-15T03:48:18.935+00:00", "t": 555.249, "phase": "create_model.done", "model_id": "ddb4615c-4bb0-44b0-92a1-c79e668c7324"} -{"ts": "2026-10-15T03:48:21.731+00:00", "t": 558.046, "phase": "forward_backward.start"} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log deleted file mode 100644 index a12eda14..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/client.log +++ /dev/null @@ -1,6 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.001s create_model.start -[cold-start] 555.249s create_model.done {'model_id': 'ddb4615c-4bb0-44b0-92a1-c79e668c7324'} -[cold-start] 558.046s forward_backward.start -Retrying request self.request_id='fc48f953-c5c6-4169-b997-c905ba50b668' because of try_again -EXIT=143 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt deleted file mode 100644 index a3312864..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T04:27:09.940+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json deleted file mode 100644 index d1cfc5a9..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/events.json +++ /dev/null @@ -1,401 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:31:11Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071080495023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:31:11Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T03:31:11Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96671460e727", - "namespace": "openrl-system", - "resourceVersion": "1792035071245135014", - "uid": "8acbbf2e-49f9-4202-949a-77c580789a9a" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:31:15Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:31:15Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", - "metadata": { - "creationTimestamp": "2026-10-15T03:31:15Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9668063dcb02", - "namespace": "openrl-system", - "resourceVersion": "1792035075278831019", - "uid": "6912eb66-4b94-4b18-a74e-ab327aeddb23" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:32:50Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:32:50Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m35.306s (1m35.306s including waiting). Image size: 7123551739 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T03:32:50Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e36f0cbc2", - "namespace": "openrl-system", - "resourceVersion": "1792035170612415015", - "uid": "8591c4e1-a15e-4174-ad4a-fd97a845ccf1" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:32:50Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:32:50Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T03:32:50Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3778892c", - "namespace": "openrl-system", - "resourceVersion": "1792035170630543005", - "uid": "ffa2ae1f-d67a-48f5-bbae-e60a320c7ca4" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:32:50Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:32:50Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T03:32:50Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3f40302b", - "namespace": "openrl-system", - "resourceVersion": "1792035170729087005", - "uid": "1519861d-65be-4db6-b777-bd8220ced5f8" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:38:15Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:38:15Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:38:15Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96c9e6a9a8f5", - "namespace": "openrl-system", - "resourceVersion": "1792035495672207024", - "uid": "ddca643d-0c58-4430-9459-5686015bcf61" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:38:15Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792034054934527006", - "uid": "5f7199d4-14aa-4747-9728-d940f5af679f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:38:15Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:38:15Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96c9ee8c225c", - "namespace": "openrl-system", - "resourceVersion": "1792035495812399014", - "uid": "ea4fdc32-2c46-41b7-8537-f60b2a3e7ce3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:04Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544745343006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:04Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:04Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d55baf7ea2", - "namespace": "openrl-system", - "resourceVersion": "1792035544868287013", - "uid": "717619c9-bc6f-4410-89cb-933289a9091a" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:08Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:08Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:08Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d6291f5cc5", - "namespace": "openrl-system", - "resourceVersion": "1792035548309503024", - "uid": "0ad1918b-487b-45d0-95f6-90606fb0f996" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:08Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:08Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:08Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d629aa8317", - "namespace": "openrl-system", - "resourceVersion": "1792035548331599007", - "uid": "a146313b-77f3-450f-9b9d-6d0b1f04c6cf" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:08Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:08Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:08Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d62f39a7f5", - "namespace": "openrl-system", - "resourceVersion": "1792035548412447000", - "uid": "fcf01033-06e2-4cbd-a2c7-51bf347ddf42" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:45:24Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:45:24Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:45:24Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de972dcf359e7a", - "namespace": "openrl-system", - "resourceVersion": "1792035924784175002", - "uid": "b8207d84-11ad-4a25-867c-7be9b78ba580" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json deleted file mode 100644 index f940f10c..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-after.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "gke-open-rl-dra-default-pool-095bf129-0flt": [], - "gke-open-rl-dra-default-pool-c19e0101-2bk0": [ - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83" - ], - "gke-open-rl-dra-default-pool-ec729728-6mpx": [], - "gke-open-rl-dra-h100-sampler-a721f127-0853": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-h100-sampler-a721f127-ed3t": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-client:pr271-210aee9", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-h100-trainer-702095b4-1g3a": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" - ], - "gke-open-rl-dra-h100-trainer-702095b4-tuqu": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6" - ], - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" - ] -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json deleted file mode 100644 index f940f10c..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-images-before.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "gke-open-rl-dra-default-pool-095bf129-0flt": [], - "gke-open-rl-dra-default-pool-c19e0101-2bk0": [ - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83" - ], - "gke-open-rl-dra-default-pool-ec729728-6mpx": [], - "gke-open-rl-dra-h100-sampler-a721f127-0853": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-h100-sampler-a721f127-ed3t": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-client:pr271-210aee9", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-h100-trainer-702095b4-1g3a": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" - ], - "gke-open-rl-dra-h100-trainer-702095b4-tuqu": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6" - ], - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/lwr-client:20260920-0009", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:b95b249", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:pr271-210aee9" - ], - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8": [ - "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13" - ] -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt deleted file mode 100644 index d3c42940..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/node-labels-during.txt +++ /dev/null @@ -1,8 +0,0 @@ -gke-open-rl-dra-h100-sampler-a721f127-0853 Ready 9d v1.35.7-gke.1222000 true -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 25h v1.35.8-gke.1380001 true -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 25h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json deleted file mode 100644 index 5bfc412e..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/pods.json +++ /dev/null @@ -1,5980 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-api-server-645ff9777d-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "645ff9777d", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - } - ], - "resourceVersion": "1791941173996783021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-79xdl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:02Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:26:02Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.10", - "podIPs": [ - { - "ip": "10.32.3.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log deleted file mode 100644 index 74402189..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/redis-ready.log +++ /dev/null @@ -1,1001 +0,0 @@ -2026-10-15T03:39:00.557+00:00 2026-10-15T03:39:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:00.558+00:00 2026-10-15T03:39:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:01.562+00:00 2026-10-15T03:39:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:01.562+00:00 2026-10-15T03:39:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:02.566+00:00 2026-10-15T03:39:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:02.566+00:00 2026-10-15T03:39:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:03.572+00:00 2026-10-15T03:39:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:03.572+00:00 2026-10-15T03:39:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:04.576+00:00 2026-10-15T03:39:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:04.576+00:00 2026-10-15T03:39:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:05.582+00:00 2026-10-15T03:39:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:05.582+00:00 2026-10-15T03:39:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:06.587+00:00 2026-10-15T03:39:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:06.587+00:00 2026-10-15T03:39:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:07.595+00:00 2026-10-15T03:39:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:07.596+00:00 2026-10-15T03:39:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:08.597+00:00 2026-10-15T03:39:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:08.597+00:00 2026-10-15T03:39:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:09.603+00:00 2026-10-15T03:39:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:09.603+00:00 2026-10-15T03:39:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:10.608+00:00 2026-10-15T03:39:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:10.608+00:00 2026-10-15T03:39:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:11.613+00:00 2026-10-15T03:39:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:11.613+00:00 2026-10-15T03:39:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:12.619+00:00 2026-10-15T03:39:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:12.619+00:00 2026-10-15T03:39:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:13.625+00:00 2026-10-15T03:39:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:13.625+00:00 2026-10-15T03:39:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:14.630+00:00 2026-10-15T03:39:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:14.630+00:00 2026-10-15T03:39:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:15.634+00:00 2026-10-15T03:39:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:15.634+00:00 2026-10-15T03:39:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:16.638+00:00 2026-10-15T03:39:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:16.639+00:00 2026-10-15T03:39:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:17.645+00:00 2026-10-15T03:39:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:17.645+00:00 2026-10-15T03:39:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:18.648+00:00 2026-10-15T03:39:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:18.648+00:00 2026-10-15T03:39:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:19.654+00:00 2026-10-15T03:39:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:19.655+00:00 2026-10-15T03:39:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:20.659+00:00 2026-10-15T03:39:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:20.659+00:00 2026-10-15T03:39:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:21.664+00:00 2026-10-15T03:39:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:21.664+00:00 2026-10-15T03:39:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:22.671+00:00 2026-10-15T03:39:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:22.672+00:00 2026-10-15T03:39:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:23.673+00:00 2026-10-15T03:39:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:23.673+00:00 2026-10-15T03:39:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:24.678+00:00 2026-10-15T03:39:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:24.678+00:00 2026-10-15T03:39:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:25.684+00:00 2026-10-15T03:39:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:25.684+00:00 2026-10-15T03:39:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:26.689+00:00 2026-10-15T03:39:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:26.689+00:00 2026-10-15T03:39:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:27.692+00:00 2026-10-15T03:39:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:27.692+00:00 2026-10-15T03:39:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:28.698+00:00 2026-10-15T03:39:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:28.699+00:00 2026-10-15T03:39:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:29.702+00:00 2026-10-15T03:39:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:29.702+00:00 2026-10-15T03:39:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:30.707+00:00 2026-10-15T03:39:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:30.707+00:00 2026-10-15T03:39:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:31.714+00:00 2026-10-15T03:39:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:31.714+00:00 2026-10-15T03:39:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:32.717+00:00 2026-10-15T03:39:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:32.717+00:00 2026-10-15T03:39:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:33.722+00:00 2026-10-15T03:39:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:33.722+00:00 2026-10-15T03:39:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:34.726+00:00 2026-10-15T03:39:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:34.726+00:00 2026-10-15T03:39:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:35.731+00:00 2026-10-15T03:39:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:35.731+00:00 2026-10-15T03:39:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:36.749+00:00 2026-10-15T03:39:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:36.749+00:00 2026-10-15T03:39:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:37.742+00:00 2026-10-15T03:39:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:37.742+00:00 2026-10-15T03:39:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:38.746+00:00 2026-10-15T03:39:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:38.746+00:00 2026-10-15T03:39:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:39.751+00:00 2026-10-15T03:39:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:39.751+00:00 2026-10-15T03:39:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:40.756+00:00 2026-10-15T03:39:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:40.756+00:00 2026-10-15T03:39:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:41.761+00:00 2026-10-15T03:39:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:41.762+00:00 2026-10-15T03:39:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:42.777+00:00 2026-10-15T03:39:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:42.777+00:00 2026-10-15T03:39:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:43.772+00:00 2026-10-15T03:39:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:43.772+00:00 2026-10-15T03:39:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:44.776+00:00 2026-10-15T03:39:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:44.777+00:00 2026-10-15T03:39:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:45.785+00:00 2026-10-15T03:39:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:45.785+00:00 2026-10-15T03:39:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:46.786+00:00 2026-10-15T03:39:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:46.786+00:00 2026-10-15T03:39:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:47.791+00:00 2026-10-15T03:39:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:47.791+00:00 2026-10-15T03:39:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:48.796+00:00 2026-10-15T03:39:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:48.796+00:00 2026-10-15T03:39:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:49.801+00:00 2026-10-15T03:39:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:49.801+00:00 2026-10-15T03:39:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:50.815+00:00 2026-10-15T03:39:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:50.815+00:00 2026-10-15T03:39:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:51.900+00:00 2026-10-15T03:39:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:51.900+00:00 2026-10-15T03:39:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:52.816+00:00 2026-10-15T03:39:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:52.816+00:00 2026-10-15T03:39:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:53.820+00:00 2026-10-15T03:39:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:53.820+00:00 2026-10-15T03:39:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:54.826+00:00 2026-10-15T03:39:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:54.827+00:00 2026-10-15T03:39:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:55.831+00:00 2026-10-15T03:39:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:55.831+00:00 2026-10-15T03:39:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:56.836+00:00 2026-10-15T03:39:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:56.837+00:00 2026-10-15T03:39:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:57.841+00:00 2026-10-15T03:39:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:57.841+00:00 2026-10-15T03:39:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:58.846+00:00 2026-10-15T03:39:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:58.846+00:00 2026-10-15T03:39:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:39:59.850+00:00 2026-10-15T03:39:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:39:59.850+00:00 2026-10-15T03:39:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:00.855+00:00 2026-10-15T03:40:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:00.855+00:00 2026-10-15T03:40:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:01.861+00:00 2026-10-15T03:40:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:01.861+00:00 2026-10-15T03:40:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:02.867+00:00 2026-10-15T03:40:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:02.867+00:00 2026-10-15T03:40:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:03.871+00:00 2026-10-15T03:40:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:03.871+00:00 2026-10-15T03:40:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:04.876+00:00 2026-10-15T03:40:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:04.876+00:00 2026-10-15T03:40:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:05.933+00:00 2026-10-15T03:40:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:05.933+00:00 2026-10-15T03:40:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:06.886+00:00 2026-10-15T03:40:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:06.886+00:00 2026-10-15T03:40:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:07.890+00:00 2026-10-15T03:40:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:07.890+00:00 2026-10-15T03:40:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:08.895+00:00 2026-10-15T03:40:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:08.895+00:00 2026-10-15T03:40:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:09.900+00:00 2026-10-15T03:40:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:09.900+00:00 2026-10-15T03:40:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:10.939+00:00 2026-10-15T03:40:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:10.940+00:00 2026-10-15T03:40:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:11.910+00:00 2026-10-15T03:40:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:11.910+00:00 2026-10-15T03:40:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:12.915+00:00 2026-10-15T03:40:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:12.915+00:00 2026-10-15T03:40:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:13.920+00:00 2026-10-15T03:40:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:13.920+00:00 2026-10-15T03:40:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:14.927+00:00 2026-10-15T03:40:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:14.927+00:00 2026-10-15T03:40:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:15.930+00:00 2026-10-15T03:40:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:15.930+00:00 2026-10-15T03:40:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:16.942+00:00 2026-10-15T03:40:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:16.942+00:00 2026-10-15T03:40:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:17.942+00:00 2026-10-15T03:40:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:17.942+00:00 2026-10-15T03:40:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:18.945+00:00 2026-10-15T03:40:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:18.946+00:00 2026-10-15T03:40:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:19.953+00:00 2026-10-15T03:40:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:19.953+00:00 2026-10-15T03:40:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:20.956+00:00 2026-10-15T03:40:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:20.956+00:00 2026-10-15T03:40:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:22.010+00:00 2026-10-15T03:40:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:22.011+00:00 2026-10-15T03:40:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:22.966+00:00 2026-10-15T03:40:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:22.966+00:00 2026-10-15T03:40:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:23.972+00:00 2026-10-15T03:40:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:23.972+00:00 2026-10-15T03:40:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:24.976+00:00 2026-10-15T03:40:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:24.976+00:00 2026-10-15T03:40:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:25.981+00:00 2026-10-15T03:40:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:25.981+00:00 2026-10-15T03:40:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:26.985+00:00 2026-10-15T03:40:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:26.985+00:00 2026-10-15T03:40:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:27.990+00:00 2026-10-15T03:40:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:27.990+00:00 2026-10-15T03:40:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:28.995+00:00 2026-10-15T03:40:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:28.995+00:00 2026-10-15T03:40:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:30.003+00:00 2026-10-15T03:40:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:30.003+00:00 2026-10-15T03:40:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:31.005+00:00 2026-10-15T03:40:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:31.006+00:00 2026-10-15T03:40:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:32.012+00:00 2026-10-15T03:40:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:32.012+00:00 2026-10-15T03:40:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:33.017+00:00 2026-10-15T03:40:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:33.017+00:00 2026-10-15T03:40:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:34.022+00:00 2026-10-15T03:40:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:34.022+00:00 2026-10-15T03:40:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:35.027+00:00 2026-10-15T03:40:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:35.027+00:00 2026-10-15T03:40:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:36.033+00:00 2026-10-15T03:40:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:36.033+00:00 2026-10-15T03:40:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:37.038+00:00 2026-10-15T03:40:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:37.038+00:00 2026-10-15T03:40:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:38.043+00:00 2026-10-15T03:40:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:38.043+00:00 2026-10-15T03:40:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:39.048+00:00 2026-10-15T03:40:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:39.048+00:00 2026-10-15T03:40:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:40.053+00:00 2026-10-15T03:40:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:40.053+00:00 2026-10-15T03:40:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:41.057+00:00 2026-10-15T03:40:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:41.057+00:00 2026-10-15T03:40:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:42.063+00:00 2026-10-15T03:40:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:42.063+00:00 2026-10-15T03:40:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:43.093+00:00 2026-10-15T03:40:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:43.094+00:00 2026-10-15T03:40:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:44.080+00:00 2026-10-15T03:40:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:44.080+00:00 2026-10-15T03:40:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:45.078+00:00 2026-10-15T03:40:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:45.078+00:00 2026-10-15T03:40:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:46.083+00:00 2026-10-15T03:40:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:46.083+00:00 2026-10-15T03:40:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:47.089+00:00 2026-10-15T03:40:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:47.089+00:00 2026-10-15T03:40:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:48.098+00:00 2026-10-15T03:40:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:48.098+00:00 2026-10-15T03:40:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:49.097+00:00 2026-10-15T03:40:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:49.098+00:00 2026-10-15T03:40:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:50.103+00:00 2026-10-15T03:40:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:50.103+00:00 2026-10-15T03:40:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:51.221+00:00 2026-10-15T03:40:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:51.221+00:00 2026-10-15T03:40:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:52.121+00:00 2026-10-15T03:40:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:52.121+00:00 2026-10-15T03:40:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:53.121+00:00 2026-10-15T03:40:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:53.121+00:00 2026-10-15T03:40:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:54.126+00:00 2026-10-15T03:40:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:54.127+00:00 2026-10-15T03:40:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:55.130+00:00 2026-10-15T03:40:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:55.130+00:00 2026-10-15T03:40:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:56.230+00:00 2026-10-15T03:40:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:56.230+00:00 2026-10-15T03:40:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:57.141+00:00 2026-10-15T03:40:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:57.141+00:00 2026-10-15T03:40:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:58.146+00:00 2026-10-15T03:40:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:58.146+00:00 2026-10-15T03:40:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:40:59.152+00:00 2026-10-15T03:40:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:40:59.152+00:00 2026-10-15T03:40:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:00.162+00:00 2026-10-15T03:41:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:00.162+00:00 2026-10-15T03:41:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:01.162+00:00 2026-10-15T03:41:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:01.162+00:00 2026-10-15T03:41:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:02.166+00:00 2026-10-15T03:41:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:02.167+00:00 2026-10-15T03:41:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:03.172+00:00 2026-10-15T03:41:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:03.172+00:00 2026-10-15T03:41:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:04.176+00:00 2026-10-15T03:41:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:04.176+00:00 2026-10-15T03:41:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:05.181+00:00 2026-10-15T03:41:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:05.181+00:00 2026-10-15T03:41:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:06.187+00:00 2026-10-15T03:41:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:06.187+00:00 2026-10-15T03:41:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:07.192+00:00 2026-10-15T03:41:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:07.192+00:00 2026-10-15T03:41:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:08.197+00:00 2026-10-15T03:41:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:08.197+00:00 2026-10-15T03:41:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:09.200+00:00 2026-10-15T03:41:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:09.201+00:00 2026-10-15T03:41:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:10.206+00:00 2026-10-15T03:41:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:10.206+00:00 2026-10-15T03:41:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:11.211+00:00 2026-10-15T03:41:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:11.211+00:00 2026-10-15T03:41:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:12.216+00:00 2026-10-15T03:41:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:12.216+00:00 2026-10-15T03:41:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:13.221+00:00 2026-10-15T03:41:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:13.221+00:00 2026-10-15T03:41:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:14.227+00:00 2026-10-15T03:41:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:14.227+00:00 2026-10-15T03:41:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:15.232+00:00 2026-10-15T03:41:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:15.232+00:00 2026-10-15T03:41:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:16.236+00:00 2026-10-15T03:41:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:16.236+00:00 2026-10-15T03:41:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:17.241+00:00 2026-10-15T03:41:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:17.241+00:00 2026-10-15T03:41:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:18.247+00:00 2026-10-15T03:41:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:18.247+00:00 2026-10-15T03:41:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:19.251+00:00 2026-10-15T03:41:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:19.251+00:00 2026-10-15T03:41:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:20.257+00:00 2026-10-15T03:41:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:20.257+00:00 2026-10-15T03:41:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:21.261+00:00 2026-10-15T03:41:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:21.261+00:00 2026-10-15T03:41:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:22.267+00:00 2026-10-15T03:41:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:22.267+00:00 2026-10-15T03:41:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:23.272+00:00 2026-10-15T03:41:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:23.272+00:00 2026-10-15T03:41:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:24.276+00:00 2026-10-15T03:41:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:24.276+00:00 2026-10-15T03:41:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:25.283+00:00 2026-10-15T03:41:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:25.283+00:00 2026-10-15T03:41:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:26.286+00:00 2026-10-15T03:41:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:26.286+00:00 2026-10-15T03:41:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:27.292+00:00 2026-10-15T03:41:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:27.292+00:00 2026-10-15T03:41:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:28.297+00:00 2026-10-15T03:41:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:28.297+00:00 2026-10-15T03:41:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:29.303+00:00 2026-10-15T03:41:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:29.303+00:00 2026-10-15T03:41:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:30.308+00:00 2026-10-15T03:41:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:30.308+00:00 2026-10-15T03:41:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:31.314+00:00 2026-10-15T03:41:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:31.314+00:00 2026-10-15T03:41:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:32.319+00:00 2026-10-15T03:41:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:32.320+00:00 2026-10-15T03:41:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:33.324+00:00 2026-10-15T03:41:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:33.324+00:00 2026-10-15T03:41:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:34.329+00:00 2026-10-15T03:41:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:34.329+00:00 2026-10-15T03:41:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:35.334+00:00 2026-10-15T03:41:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:35.334+00:00 2026-10-15T03:41:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:36.346+00:00 2026-10-15T03:41:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:36.346+00:00 2026-10-15T03:41:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:37.344+00:00 2026-10-15T03:41:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:37.344+00:00 2026-10-15T03:41:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:38.349+00:00 2026-10-15T03:41:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:38.349+00:00 2026-10-15T03:41:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:39.353+00:00 2026-10-15T03:41:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:39.353+00:00 2026-10-15T03:41:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:40.358+00:00 2026-10-15T03:41:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:40.358+00:00 2026-10-15T03:41:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:41.363+00:00 2026-10-15T03:41:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:41.363+00:00 2026-10-15T03:41:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:42.369+00:00 2026-10-15T03:41:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:42.369+00:00 2026-10-15T03:41:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:43.373+00:00 2026-10-15T03:41:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:43.373+00:00 2026-10-15T03:41:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:44.378+00:00 2026-10-15T03:41:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:44.378+00:00 2026-10-15T03:41:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:45.383+00:00 2026-10-15T03:41:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:45.383+00:00 2026-10-15T03:41:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:46.388+00:00 2026-10-15T03:41:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:46.388+00:00 2026-10-15T03:41:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:47.392+00:00 2026-10-15T03:41:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:47.392+00:00 2026-10-15T03:41:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:48.397+00:00 2026-10-15T03:41:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:48.397+00:00 2026-10-15T03:41:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:49.404+00:00 2026-10-15T03:41:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:49.404+00:00 2026-10-15T03:41:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:50.407+00:00 2026-10-15T03:41:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:50.407+00:00 2026-10-15T03:41:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:51.411+00:00 2026-10-15T03:41:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:51.412+00:00 2026-10-15T03:41:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:52.417+00:00 2026-10-15T03:41:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:52.417+00:00 2026-10-15T03:41:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:53.422+00:00 2026-10-15T03:41:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:53.422+00:00 2026-10-15T03:41:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:54.427+00:00 2026-10-15T03:41:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:54.427+00:00 2026-10-15T03:41:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:55.433+00:00 2026-10-15T03:41:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:55.433+00:00 2026-10-15T03:41:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:56.455+00:00 2026-10-15T03:41:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:56.455+00:00 2026-10-15T03:41:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:57.443+00:00 2026-10-15T03:41:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:57.443+00:00 2026-10-15T03:41:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:58.447+00:00 2026-10-15T03:41:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:58.447+00:00 2026-10-15T03:41:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:41:59.453+00:00 2026-10-15T03:41:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:41:59.454+00:00 2026-10-15T03:41:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:00.457+00:00 2026-10-15T03:42:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:00.457+00:00 2026-10-15T03:42:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:01.486+00:00 2026-10-15T03:42:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:01.486+00:00 2026-10-15T03:42:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:02.468+00:00 2026-10-15T03:42:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:02.468+00:00 2026-10-15T03:42:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:03.473+00:00 2026-10-15T03:42:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:03.473+00:00 2026-10-15T03:42:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:04.477+00:00 2026-10-15T03:42:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:04.477+00:00 2026-10-15T03:42:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:05.483+00:00 2026-10-15T03:42:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:05.483+00:00 2026-10-15T03:42:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:06.487+00:00 2026-10-15T03:42:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:06.487+00:00 2026-10-15T03:42:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:07.492+00:00 2026-10-15T03:42:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:07.492+00:00 2026-10-15T03:42:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:08.498+00:00 2026-10-15T03:42:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:08.499+00:00 2026-10-15T03:42:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:09.502+00:00 2026-10-15T03:42:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:09.502+00:00 2026-10-15T03:42:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:10.512+00:00 2026-10-15T03:42:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:10.512+00:00 2026-10-15T03:42:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:11.513+00:00 2026-10-15T03:42:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:11.513+00:00 2026-10-15T03:42:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:12.518+00:00 2026-10-15T03:42:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:12.518+00:00 2026-10-15T03:42:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:13.523+00:00 2026-10-15T03:42:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:13.523+00:00 2026-10-15T03:42:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:14.528+00:00 2026-10-15T03:42:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:14.528+00:00 2026-10-15T03:42:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:15.535+00:00 2026-10-15T03:42:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:15.535+00:00 2026-10-15T03:42:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:16.538+00:00 2026-10-15T03:42:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:16.538+00:00 2026-10-15T03:42:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:17.547+00:00 2026-10-15T03:42:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:17.547+00:00 2026-10-15T03:42:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:18.550+00:00 2026-10-15T03:42:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:18.550+00:00 2026-10-15T03:42:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:19.566+00:00 2026-10-15T03:42:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:19.566+00:00 2026-10-15T03:42:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:20.574+00:00 2026-10-15T03:42:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:20.574+00:00 2026-10-15T03:42:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:21.568+00:00 2026-10-15T03:42:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:21.568+00:00 2026-10-15T03:42:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:22.573+00:00 2026-10-15T03:42:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:22.574+00:00 2026-10-15T03:42:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:23.586+00:00 2026-10-15T03:42:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:23.586+00:00 2026-10-15T03:42:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:24.583+00:00 2026-10-15T03:42:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:24.584+00:00 2026-10-15T03:42:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:25.589+00:00 2026-10-15T03:42:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:25.589+00:00 2026-10-15T03:42:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:26.594+00:00 2026-10-15T03:42:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:26.594+00:00 2026-10-15T03:42:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:27.599+00:00 2026-10-15T03:42:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:27.599+00:00 2026-10-15T03:42:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:28.610+00:00 2026-10-15T03:42:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:28.610+00:00 2026-10-15T03:42:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:29.610+00:00 2026-10-15T03:42:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:29.610+00:00 2026-10-15T03:42:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:30.615+00:00 2026-10-15T03:42:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:30.615+00:00 2026-10-15T03:42:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:31.620+00:00 2026-10-15T03:42:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:31.621+00:00 2026-10-15T03:42:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:32.626+00:00 2026-10-15T03:42:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:32.626+00:00 2026-10-15T03:42:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:33.646+00:00 2026-10-15T03:42:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:33.646+00:00 2026-10-15T03:42:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:34.636+00:00 2026-10-15T03:42:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:34.636+00:00 2026-10-15T03:42:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:35.641+00:00 2026-10-15T03:42:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:35.641+00:00 2026-10-15T03:42:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:36.647+00:00 2026-10-15T03:42:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:36.647+00:00 2026-10-15T03:42:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:37.654+00:00 2026-10-15T03:42:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:37.654+00:00 2026-10-15T03:42:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:38.657+00:00 2026-10-15T03:42:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:38.657+00:00 2026-10-15T03:42:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:39.672+00:00 2026-10-15T03:42:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:39.672+00:00 2026-10-15T03:42:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:40.667+00:00 2026-10-15T03:42:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:40.667+00:00 2026-10-15T03:42:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:41.673+00:00 2026-10-15T03:42:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:41.673+00:00 2026-10-15T03:42:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:42.676+00:00 2026-10-15T03:42:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:42.676+00:00 2026-10-15T03:42:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:43.681+00:00 2026-10-15T03:42:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:43.682+00:00 2026-10-15T03:42:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:44.686+00:00 2026-10-15T03:42:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:44.687+00:00 2026-10-15T03:42:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:45.691+00:00 2026-10-15T03:42:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:45.691+00:00 2026-10-15T03:42:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:46.698+00:00 2026-10-15T03:42:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:46.698+00:00 2026-10-15T03:42:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:47.702+00:00 2026-10-15T03:42:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:47.702+00:00 2026-10-15T03:42:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:48.708+00:00 2026-10-15T03:42:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:48.708+00:00 2026-10-15T03:42:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:49.712+00:00 2026-10-15T03:42:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:49.712+00:00 2026-10-15T03:42:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:50.721+00:00 2026-10-15T03:42:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:50.721+00:00 2026-10-15T03:42:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:51.728+00:00 2026-10-15T03:42:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:51.728+00:00 2026-10-15T03:42:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:52.727+00:00 2026-10-15T03:42:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:52.727+00:00 2026-10-15T03:42:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:53.732+00:00 2026-10-15T03:42:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:53.732+00:00 2026-10-15T03:42:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:54.737+00:00 2026-10-15T03:42:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:54.737+00:00 2026-10-15T03:42:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:55.742+00:00 2026-10-15T03:42:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:55.742+00:00 2026-10-15T03:42:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:56.747+00:00 2026-10-15T03:42:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:56.747+00:00 2026-10-15T03:42:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:57.762+00:00 2026-10-15T03:42:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:57.762+00:00 2026-10-15T03:42:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:58.757+00:00 2026-10-15T03:42:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:58.757+00:00 2026-10-15T03:42:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:42:59.762+00:00 2026-10-15T03:42:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:42:59.762+00:00 2026-10-15T03:42:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:00.773+00:00 2026-10-15T03:43:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:00.773+00:00 2026-10-15T03:43:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:01.776+00:00 2026-10-15T03:43:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:01.776+00:00 2026-10-15T03:43:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:02.777+00:00 2026-10-15T03:43:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:02.777+00:00 2026-10-15T03:43:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:03.782+00:00 2026-10-15T03:43:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:03.782+00:00 2026-10-15T03:43:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:04.789+00:00 2026-10-15T03:43:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:04.789+00:00 2026-10-15T03:43:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:05.792+00:00 2026-10-15T03:43:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:05.792+00:00 2026-10-15T03:43:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:06.796+00:00 2026-10-15T03:43:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:06.796+00:00 2026-10-15T03:43:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:07.802+00:00 2026-10-15T03:43:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:07.802+00:00 2026-10-15T03:43:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:08.806+00:00 2026-10-15T03:43:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:08.806+00:00 2026-10-15T03:43:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:09.810+00:00 2026-10-15T03:43:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:09.810+00:00 2026-10-15T03:43:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:10.816+00:00 2026-10-15T03:43:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:10.816+00:00 2026-10-15T03:43:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:11.822+00:00 2026-10-15T03:43:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:11.822+00:00 2026-10-15T03:43:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:12.826+00:00 2026-10-15T03:43:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:12.826+00:00 2026-10-15T03:43:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:13.831+00:00 2026-10-15T03:43:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:13.831+00:00 2026-10-15T03:43:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:14.836+00:00 2026-10-15T03:43:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:14.836+00:00 2026-10-15T03:43:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:15.842+00:00 2026-10-15T03:43:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:15.842+00:00 2026-10-15T03:43:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:43:16.847+00:00 2026-10-15T03:43:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:43:16.847+00:00 2026-10-15T03:43:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.694+00:00 2026-10-15T03:43:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.695+00:00 2026-10-15T03:43:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.695+00:00 2026-10-15T03:43:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.695+00:00 2026-10-15T03:43:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.745+00:00 2026-10-15T03:43:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.749+00:00 2026-10-15T03:43:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.749+00:00 2026-10-15T03:43:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.806+00:00 2026-10-15T03:43:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.806+00:00 2026-10-15T03:43:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.813+00:00 2026-10-15T03:43:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.813+00:00 2026-10-15T03:43:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.851+00:00 2026-10-15T03:43:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.851+00:00 2026-10-15T03:43:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.861+00:00 2026-10-15T03:43:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.861+00:00 2026-10-15T03:43:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.902+00:00 2026-10-15T03:43:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.902+00:00 2026-10-15T03:43:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.910+00:00 2026-10-15T03:43:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.951+00:00 2026-10-15T03:43:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.951+00:00 2026-10-15T03:43:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:04.955+00:00 2026-10-15T03:43:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.002+00:00 2026-10-15T03:43:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.002+00:00 2026-10-15T03:43:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.011+00:00 2026-10-15T03:43:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.062+00:00 2026-10-15T03:43:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.062+00:00 2026-10-15T03:43:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.068+00:00 2026-10-15T03:43:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.126+00:00 2026-10-15T03:43:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.127+00:00 2026-10-15T03:43:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.185+00:00 2026-10-15T03:43:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:43:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.186+00:00 2026-10-15T03:44:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.221+00:00 2026-10-15T03:44:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.234+00:00 2026-10-15T03:44:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.240+00:00 2026-10-15T03:44:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:44:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.241+00:00 2026-10-15T03:45:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.242+00:00 2026-10-15T03:45:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.292+00:00 2026-10-15T03:45:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.292+00:00 2026-10-15T03:45:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.293+00:00 2026-10-15T03:45:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:45:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.294+00:00 2026-10-15T03:46:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.342+00:00 2026-10-15T03:46:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.344+00:00 2026-10-15T03:46:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:46:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:46:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:46:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.345+00:00 2026-10-15T03:47:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:47:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.346+00:00 2026-10-15T03:48:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.392+00:00 2026-10-15T03:48:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.393+00:00 2026-10-15T03:48:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.393+00:00 2026-10-15T03:48:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.393+00:00 2026-10-15T03:48:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.394+00:00 2026-10-15T03:48:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:48:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.396+00:00 2026-10-15T03:49:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:05.614+00:00 2026-10-15T03:49:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:06.622+00:00 2026-10-15T03:49:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:07.626+00:00 2026-10-15T03:49:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:08.631+00:00 2026-10-15T03:49:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:09.637+00:00 2026-10-15T03:49:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:10.643+00:00 2026-10-15T03:49:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:11.646+00:00 2026-10-15T03:49:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:12.651+00:00 2026-10-15T03:49:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:13.657+00:00 2026-10-15T03:49:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:14.661+00:00 2026-10-15T03:49:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:15.666+00:00 2026-10-15T03:49:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:16.671+00:00 2026-10-15T03:49:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T03:49:17.676+00:00 2026-10-15T03:49:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:05:57.232+00:00 E1014 21:05:57.231723 5403 v2.go:150] "Unhandled Error" err="next reader: read tcp 192.168.1.37:57575->136.116.158.163:443: read: connection reset by peer" -2026-10-15T04:05:57.233+00:00 E1014 21:05:57.231748 5403 v2.go:167] "Unhandled Error" err="next reader: read tcp 192.168.1.37:57575->136.116.158.163:443: read: connection reset by peer" -2026-10-15T04:05:57.234+00:00 error: error reading from error stream: next reader: read tcp 192.168.1.37:57575->136.116.158.163:443: read: connection reset by peer diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log deleted file mode 100644 index 30f82985..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/run.log +++ /dev/null @@ -1,36 +0,0 @@ -watching; recorders [5401, 5402, 5404, 5406, 5408, 5403, 5405, 5407] write into runs/2026-10-15_cold_start/D-8b-h100-cached -collected 0 worker log(s) into runs/2026-10-15_cold_start/D-8b-h100-cached -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | unmeasured | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | unmeasured | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | unmeasured | -| container create | trainer.event_Pulled | trainer.container_started | unmeasured | -| process start to load start (imports) | trainer.container_started | trainer.load_start | unmeasured | -| weights load | trainer.load_start | trainer.load_done | unmeasured | -| ready key | trainer.load_done | trainer.ready_key | unmeasured | -| client: create_model blocked | client.create_model.start | client.create_model.done | 555.2 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | unmeasured | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | unmeasured | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | unmeasured | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | unmeasured | -| container create | sampler.event_Pulled | sampler.container_started | unmeasured | -| process start to first log | sampler.container_started | sampler.process_first_log | unmeasured | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | unmeasured | -| engine init until load start | sampler.engine_init_start | sampler.load_start | unmeasured | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | unmeasured | -| torch.compile | sampler.weights_loaded | sampler.compile_done | unmeasured | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | unmeasured | -| ready key | sampler.engine_init_done | sampler.ready_key | unmeasured | -| client: first sample blocked | client.sample.start | client.sample.done | unmeasured | - -DONE=0 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log deleted file mode 100644 index 8a48c226..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T03:39:04.606+00:00 2026-10-15T03:39:04.595116714Z {"level":"info","ts":"2026-10-15T03:39:04Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"991ac44f-13dc-4b3b-88a3-ea3017a1451b","claim":"claim-lora-qwen-qwen3-8b-0-trainer-f4ace73f","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} -2026-10-15T03:48:27.398+00:00 error: read tcp 192.168.1.37:57571->136.116.158.163:443: read: connection reset by peer diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json deleted file mode 100644 index 66e85a7f..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/timeline.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "case": "D-8b-h100-cached", - "collected": "2026-10-15T04:27:13.680+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 03:39:03.686000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen3-8B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 03:39:03.686000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 03:48:18.935000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "ddb4615c-4bb0-44b0-92a1-c79e668c7324" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 03:48:21.731000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": {} -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt deleted file mode 100644 index 722e1311..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T03:38:59.461+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt deleted file mode 100644 index 0d162c07..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json b/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json deleted file mode 100644 index 6da8d279..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached-interrupted/workloads.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "apiVersion": "v1", - "items": [], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached.run.log b/runs/2026-10-15_cold_start/D-8b-h100-cached.run.log deleted file mode 100644 index 1024f918..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached.run.log +++ /dev/null @@ -1,35 +0,0 @@ -watching; recorders [8878, 8879, 8881, 8883, 8885, 8880, 8882, 8884] write into runs/2026-10-15_cold_start/D-8b-h100-cached -collected 2 worker log(s) into runs/2026-10-15_cold_start/D-8b-h100-cached -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.2 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | -| container create | trainer.event_Pulled | trainer.container_started | 0.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.4 | -| weights load | trainer.load_start | trainer.load_done | 39.9 | -| ready key | trainer.load_done | trainer.ready_key | -0.3 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 52.2 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.3 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 3.0 | -| container create | sampler.event_Pulled | sampler.container_started | 1.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 7.1 | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 17.7 | -| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 13.5 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.4 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.5 | -| ready key | sampler.engine_init_done | sampler.ready_key | 0.6 | -| client: first sample blocked | client.sample.start | client.sample.done | 120.7 | - diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log deleted file mode 100644 index b2279560..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/api-server.log +++ /dev/null @@ -1,36 +0,0 @@ -2026-10-15T04:28:48.050+00:00 2026-10-15T04:28:48.048635307Z INFO: 127.0.0.1:39700 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T04:28:48.286+00:00 2026-10-15T04:28:48.286095440Z INFO: 127.0.0.1:39712 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T04:28:48.355+00:00 2026-10-15T04:28:48.350028924Z INFO: 127.0.0.1:39712 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T04:28:48.592+00:00 2026-10-15T04:28:48.593057216Z [API_SERVER] enqueued op=create_model request_id=db28c229-9bd6-4845-bd41-edb2e3018375 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 -2026-10-15T04:28:48.594+00:00 2026-10-15T04:28:48.593584627Z INFO: 127.0.0.1:39714 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T04:28:53.965+00:00 2026-10-15T04:28:53.964292215Z INFO: 10.32.3.1:47308 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:28:58.571+00:00 2026-10-15T04:28:58.568532635Z INFO: 127.0.0.1:59700 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:29:03.967+00:00 2026-10-15T04:29:03.965572350Z INFO: 10.32.3.1:49272 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:29:13.967+00:00 2026-10-15T04:29:13.965469203Z INFO: 10.32.3.1:53736 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:29:23.972+00:00 2026-10-15T04:29:23.965165004Z INFO: 10.32.3.1:57264 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:29:33.967+00:00 2026-10-15T04:29:33.965503320Z INFO: 10.32.3.1:41922 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:29:38.739+00:00 2026-10-15T04:29:38.738456425Z INFO: 127.0.0.1:58346 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:29:40.123+00:00 2026-10-15T04:29:40.120084777Z INFO: 127.0.0.1:58376 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T04:29:42.785+00:00 2026-10-15T04:29:42.783707482Z [API_SERVER] enqueued op=forward_backward request_id=5827dd5b-d046-465a-9f51-a4c5fb10be88 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 -2026-10-15T04:29:42.786+00:00 2026-10-15T04:29:42.784716161Z INFO: 127.0.0.1:58376 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T04:29:43.964+00:00 2026-10-15T04:29:43.964611788Z INFO: 10.32.3.1:36414 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:29:45.701+00:00 2026-10-15T04:29:45.690175181Z [API_SERVER] enqueued op=optim_step request_id=0bd1fc1f-9862-42b8-87a3-bb2b38d15a08 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 -2026-10-15T04:29:45.707+00:00 2026-10-15T04:29:45.690965153Z INFO: 127.0.0.1:58376 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T04:29:47.841+00:00 2026-10-15T04:29:47.840662196Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=bb1b8120-ce82-4ed2-8a6c-e0803b9cdc65 model_id=db28c229-9bd6-4845-bd41-edb2e3018375 active_set=Qwen/Qwen3-8B-1 -2026-10-15T04:29:47.841+00:00 2026-10-15T04:29:47.841190565Z INFO: 127.0.0.1:58376 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T04:29:49.846+00:00 2026-10-15T04:29:49.845879451Z INFO: 127.0.0.1:35416 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T04:29:53.966+00:00 2026-10-15T04:29:53.965198781Z INFO: 10.32.3.1:35352 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:30:03.968+00:00 2026-10-15T04:30:03.966186951Z INFO: 10.32.3.1:35492 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:30:13.967+00:00 2026-10-15T04:30:13.965413981Z INFO: 10.32.3.1:42874 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:30:23.995+00:00 2026-10-15T04:30:23.965280850Z INFO: 10.32.3.1:52978 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:30:33.965+00:00 2026-10-15T04:30:33.964533329Z INFO: 10.32.3.1:52066 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:30:38.938+00:00 2026-10-15T04:30:38.922489978Z INFO: 127.0.0.1:36472 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:30:43.967+00:00 2026-10-15T04:30:43.965005983Z INFO: 10.32.3.1:41578 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:30:53.966+00:00 2026-10-15T04:30:53.964647584Z INFO: 10.32.3.1:37766 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:31:03.966+00:00 2026-10-15T04:31:03.964948698Z INFO: 10.32.3.1:52962 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:31:13.967+00:00 2026-10-15T04:31:13.964286630Z INFO: 10.32.3.1:39666 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:31:23.967+00:00 2026-10-15T04:31:23.965294376Z INFO: 10.32.3.1:35434 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:31:29.132+00:00 2026-10-15T04:31:29.129760597Z INFO: 127.0.0.1:44156 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:31:33.966+00:00 2026-10-15T04:31:33.965082373Z INFO: 10.32.3.1:36136 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:31:43.967+00:00 2026-10-15T04:31:43.964688954Z INFO: 10.32.3.1:47940 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:31:53.966+00:00 2026-10-15T04:31:53.964597059Z INFO: 10.32.3.1:57990 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl b/runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl deleted file mode 100644 index 7a8c61f7..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T04:28:47.756+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T04:28:47.756+00:00", "t": 0.0, "phase": "create_model.start"} -{"ts": "2026-10-15T04:29:39.925+00:00", "t": 52.169, "phase": "create_model.done", "model_id": "db28c229-9bd6-4845-bd41-edb2e3018375"} -{"ts": "2026-10-15T04:29:42.710+00:00", "t": 54.955, "phase": "forward_backward.start"} -{"ts": "2026-10-15T04:29:45.623+00:00", "t": 57.867, "phase": "forward_backward.done"} -{"ts": "2026-10-15T04:29:47.732+00:00", "t": 59.976, "phase": "optim_step.done"} -{"ts": "2026-10-15T04:29:47.733+00:00", "t": 59.977, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T04:29:49.654+00:00", "t": 61.898, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T04:29:49.657+00:00", "t": 61.901, "phase": "sample.start"} -{"ts": "2026-10-15T04:31:50.315+00:00", "t": 182.559, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} -{"ts": "2026-10-15T04:31:50.316+00:00", "t": 182.56, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/client.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/client.log deleted file mode 100644 index 7c7e9887..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.000s create_model.start -[cold-start] 52.169s create_model.done {'model_id': 'db28c229-9bd6-4845-bd41-edb2e3018375'} -[cold-start] 54.955s forward_backward.start -[cold-start] 57.867s forward_backward.done -[cold-start] 59.976s optim_step.done -[cold-start] 59.977s save_weights_and_get_sampling_client.start -[cold-start] 61.898s save_weights_and_get_sampling_client.done -[cold-start] 61.901s sample.start -[cold-start] 182.559s sample.done {'text': ' 4\n\nQuestion: What is '} -[cold-start] 182.560s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt deleted file mode 100644 index 446608ec..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T04:31:50.787+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/events.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/events.json deleted file mode 100644 index 6386b858..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/events.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:31:11Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071080495023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:31:11Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T03:31:11Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96671460e727", - "namespace": "openrl-system", - "resourceVersion": "1792035071245135014", - "uid": "8acbbf2e-49f9-4202-949a-77c580789a9a" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:31:15Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:31:15Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\"", - "metadata": { - "creationTimestamp": "2026-10-15T03:31:15Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9668063dcb02", - "namespace": "openrl-system", - "resourceVersion": "1792035075278831019", - "uid": "6912eb66-4b94-4b18-a74e-ab327aeddb23" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:32:50Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:32:50Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" in 1m35.306s (1m35.306s including waiting). Image size: 7123551739 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T03:32:50Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e36f0cbc2", - "namespace": "openrl-system", - "resourceVersion": "1792035170612415015", - "uid": "8591c4e1-a15e-4174-ad4a-fd97a845ccf1" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:32:50Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:32:50Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T03:32:50Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3778892c", - "namespace": "openrl-system", - "resourceVersion": "1792035170630543005", - "uid": "ffa2ae1f-d67a-48f5-bbae-e60a320c7ca4" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:32:50Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:32:50Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T03:32:50Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de967e3f40302b", - "namespace": "openrl-system", - "resourceVersion": "1792035170729087005", - "uid": "1519861d-65be-4db6-b777-bd8220ced5f8" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:38:15Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792035071201999023", - "uid": "020b4239-66d2-4346-9d02-94db8a9199d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:38:15Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:38:15Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de96c9e6a9a8f5", - "namespace": "openrl-system", - "resourceVersion": "1792035495672207024", - "uid": "ddca643d-0c58-4430-9459-5686015bcf61" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:38:15Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792034054934527006", - "uid": "5f7199d4-14aa-4747-9728-d940f5af679f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:38:15Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:38:15Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96c9ee8c225c", - "namespace": "openrl-system", - "resourceVersion": "1792035495812399014", - "uid": "ea4fdc32-2c46-41b7-8537-f60b2a3e7ce3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:04Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544745343006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:04Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:04Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d55baf7ea2", - "namespace": "openrl-system", - "resourceVersion": "1792035544868287013", - "uid": "717619c9-bc6f-4410-89cb-933289a9091a" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:08Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:08Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:08Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d6291f5cc5", - "namespace": "openrl-system", - "resourceVersion": "1792035548309503024", - "uid": "0ad1918b-487b-45d0-95f6-90606fb0f996" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:08Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:08Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:08Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d629aa8317", - "namespace": "openrl-system", - "resourceVersion": "1792035548331599007", - "uid": "a146313b-77f3-450f-9b9d-6d0b1f04c6cf" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:39:08Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:39:08Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T03:39:08Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de96d62f39a7f5", - "namespace": "openrl-system", - "resourceVersion": "1792035548412447000", - "uid": "fcf01033-06e2-4cbd-a2c7-51bf347ddf42" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T03:45:24Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792035544844159006", - "uid": "f87406a5-2ef3-464f-8587-357cfbe07add" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T03:45:24Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T03:45:24Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de972dcf359e7a", - "namespace": "openrl-system", - "resourceVersion": "1792035924784175002", - "uid": "b8207d84-11ad-4a25-867c-7be9b78ba580" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt deleted file mode 100644 index 3a77965e..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt +++ /dev/null @@ -1,223 +0,0 @@ -2026-10-15T04:29:59.120383793Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T04:30:07.125364996Z INFO 10-15 04:30:07 [model.py:692] Resolved architecture: Qwen3ForCausalLM -2026-10-15T04:30:07.125609457Z INFO 10-15 04:30:07 [model.py:2030] Using max model len 8192 -2026-10-15T04:30:07.127331114Z INFO 10-15 04:30:07 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. -2026-10-15T04:30:07.128167407Z INFO 10-15 04:30:07 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T04:30:09.663951222Z WARNING 10-15 04:30:09 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T04:30:16.823040941Z (EngineCore pid=169) INFO 10-15 04:30:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T04:30:17.164644024Z (EngineCore pid=169) INFO 10-15 04:30:17 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_340dc35f3b0242e2bdbc2b172fcc048b backend=nccl -2026-10-15T04:30:17.168650276Z (EngineCore pid=169) INFO 10-15 04:30:17 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T04:30:17.168683746Z (EngineCore pid=169) INFO 10-15 04:30:17 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T04:30:17.937691413Z (EngineCore pid=169) INFO 10-15 04:30:17 [model_runner.py:396] Loading model from scratch... -2026-10-15T04:30:18.521439941Z (EngineCore pid=169) INFO 10-15 04:30:18 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T04:30:18.521753371Z (EngineCore pid=169) INFO 10-15 04:30:18 [flash_attn.py:1116] Using FlashAttention version 3 -2026-10-15T04:30:18.637928137Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T04:30:18.638073197Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T04:30:18.941812484Z (EngineCore pid=169) INFO 10-15 04:30:18 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. -2026-10-15T04:30:18.941839205Z (EngineCore pid=169) INFO 10-15 04:30:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T04:30:18.942611568Z (EngineCore pid=169) -Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T04:28:57.227937676Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T04:28:57.227957423Z -2026-10-15T04:28:57.431621575Z Loading base model Qwen/Qwen3-8B to cuda:0... -2026-10-15T04:28:57.548837029Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T04:29:37.083323739Z -Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json deleted file mode 100644 index 86b1422e..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-api-server-645ff9777d-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "645ff9777d", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - } - ], - "resourceVersion": "1791941173996783021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-79xdl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:02Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:26:13Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://8a3be260dfc36abcf57b4fd2fbef5bc23fa3b1d6d3f2609b32a5df81761c4274", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:agent4-e440b83", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:38e01a8e2a207f9ec9b70c545d2a3eb0da6e49ffc84149c70d705d7331d714a7", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:26:02Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-79xdl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.10", - "podIPs": [ - { - "ip": "10.32.3.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-sampler", - "uid": "aac5fdd5-1826-49c7-8392-5165024b0a69" - } - ], - "resourceVersion": "1792038592210623023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-sampler-aac5fdd5" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "64aee2d3-084c-46a4-9c77-df7f6442e580" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-vvjnl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-vvjnl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:29:52Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:29:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:29:52Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:29:52Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:29:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "40Gi" - }, - "containerID": "containerd://cb24f70562bc53e327a3c69f58bcfb325486c128f44f76ed9a396e258881cab1", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:29:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-vvjnl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.1.8", - "podIPs": [ - { - "ip": "10.32.1.8" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:29:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-trainer", - "uid": "cde28d2b-7497-4559-ae79-2c350558ddd1" - } - ], - "resourceVersion": "1792038533321423006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-trainer-cde28d2b" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "6521caef-db14-4d7b-8477-e644e69919ee" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2wss8", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2wss8", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:28:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:28:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "36Gi" - }, - "containerID": "containerd://c9a7c710fac76fa5cc69dea265bf295ddd44ae0adc3cab5d45f3adca2265857b", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2wss8", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.4.8", - "podIPs": [ - { - "ip": "10.32.4.8" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:28:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log deleted file mode 100644 index 4235b80d..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/redis-ready.log +++ /dev/null @@ -1,147 +0,0 @@ -2026-10-15T04:29:37.911+00:00 2026-10-15T04:29:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:38.916+00:00 2026-10-15T04:29:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:39.919+00:00 2026-10-15T04:29:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:40.924+00:00 2026-10-15T04:29:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:41.929+00:00 2026-10-15T04:29:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:42.941+00:00 2026-10-15T04:29:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:43.939+00:00 2026-10-15T04:29:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:44.944+00:00 2026-10-15T04:29:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:45.950+00:00 2026-10-15T04:29:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:46.953+00:00 2026-10-15T04:29:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:47.959+00:00 2026-10-15T04:29:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:48.964+00:00 2026-10-15T04:29:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:49.969+00:00 2026-10-15T04:29:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:50.975+00:00 2026-10-15T04:29:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:51.981+00:00 2026-10-15T04:29:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:52.993+00:00 2026-10-15T04:29:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:53.990+00:00 2026-10-15T04:29:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:54.995+00:00 2026-10-15T04:29:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:56.000+00:00 2026-10-15T04:29:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:57.008+00:00 2026-10-15T04:29:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:58.011+00:00 2026-10-15T04:29:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:29:59.081+00:00 2026-10-15T04:29:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:00.021+00:00 2026-10-15T04:30:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:01.026+00:00 2026-10-15T04:30:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:02.033+00:00 2026-10-15T04:30:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:03.037+00:00 2026-10-15T04:30:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:04.044+00:00 2026-10-15T04:30:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:05.047+00:00 2026-10-15T04:30:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:06.053+00:00 2026-10-15T04:30:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:07.058+00:00 2026-10-15T04:30:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:08.064+00:00 2026-10-15T04:30:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:09.069+00:00 2026-10-15T04:30:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:10.074+00:00 2026-10-15T04:30:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:11.079+00:00 2026-10-15T04:30:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:12.099+00:00 2026-10-15T04:30:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:13.090+00:00 2026-10-15T04:30:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:14.095+00:00 2026-10-15T04:30:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:15.108+00:00 2026-10-15T04:30:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:16.106+00:00 2026-10-15T04:30:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:17.114+00:00 2026-10-15T04:30:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:18.116+00:00 2026-10-15T04:30:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:19.122+00:00 2026-10-15T04:30:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:20.127+00:00 2026-10-15T04:30:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:21.132+00:00 2026-10-15T04:30:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:22.138+00:00 2026-10-15T04:30:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:23.143+00:00 2026-10-15T04:30:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:24.148+00:00 2026-10-15T04:30:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:25.153+00:00 2026-10-15T04:30:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:26.157+00:00 2026-10-15T04:30:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:27.162+00:00 2026-10-15T04:30:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:28.167+00:00 2026-10-15T04:30:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:29.171+00:00 2026-10-15T04:30:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:30.177+00:00 2026-10-15T04:30:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:31.182+00:00 2026-10-15T04:30:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:32.187+00:00 2026-10-15T04:30:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:33.195+00:00 2026-10-15T04:30:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:34.196+00:00 2026-10-15T04:30:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:35.202+00:00 2026-10-15T04:30:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:36.207+00:00 2026-10-15T04:30:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:37.212+00:00 2026-10-15T04:30:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:38.218+00:00 2026-10-15T04:30:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:39.223+00:00 2026-10-15T04:30:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:40.228+00:00 2026-10-15T04:30:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:41.234+00:00 2026-10-15T04:30:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:42.238+00:00 2026-10-15T04:30:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:43.245+00:00 2026-10-15T04:30:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:44.258+00:00 2026-10-15T04:30:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:45.254+00:00 2026-10-15T04:30:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:46.303+00:00 2026-10-15T04:30:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:47.268+00:00 2026-10-15T04:30:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:48.271+00:00 2026-10-15T04:30:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:49.275+00:00 2026-10-15T04:30:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:50.280+00:00 2026-10-15T04:30:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:51.286+00:00 2026-10-15T04:30:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:52.291+00:00 2026-10-15T04:30:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:53.296+00:00 2026-10-15T04:30:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:54.301+00:00 2026-10-15T04:30:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:55.304+00:00 2026-10-15T04:30:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:56.309+00:00 2026-10-15T04:30:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:57.314+00:00 2026-10-15T04:30:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:58.319+00:00 2026-10-15T04:30:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:30:59.415+00:00 2026-10-15T04:30:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:00.329+00:00 2026-10-15T04:31:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:01.334+00:00 2026-10-15T04:31:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:02.339+00:00 2026-10-15T04:31:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:03.344+00:00 2026-10-15T04:31:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:04.349+00:00 2026-10-15T04:31:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:05.354+00:00 2026-10-15T04:31:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:06.358+00:00 2026-10-15T04:31:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:07.363+00:00 2026-10-15T04:31:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:08.368+00:00 2026-10-15T04:31:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:09.382+00:00 2026-10-15T04:31:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:10.377+00:00 2026-10-15T04:31:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:11.382+00:00 2026-10-15T04:31:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:12.388+00:00 2026-10-15T04:31:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:13.392+00:00 2026-10-15T04:31:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:14.397+00:00 2026-10-15T04:31:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:15.401+00:00 2026-10-15T04:31:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:16.407+00:00 2026-10-15T04:31:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:17.412+00:00 2026-10-15T04:31:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:18.416+00:00 2026-10-15T04:31:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:19.423+00:00 2026-10-15T04:31:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:20.427+00:00 2026-10-15T04:31:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:21.431+00:00 2026-10-15T04:31:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:22.437+00:00 2026-10-15T04:31:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:23.454+00:00 2026-10-15T04:31:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:24.447+00:00 2026-10-15T04:31:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:25.452+00:00 2026-10-15T04:31:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:26.456+00:00 2026-10-15T04:31:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:27.461+00:00 2026-10-15T04:31:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:28.466+00:00 2026-10-15T04:31:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:29.472+00:00 2026-10-15T04:31:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:30.479+00:00 2026-10-15T04:31:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:31.482+00:00 2026-10-15T04:31:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:32.487+00:00 2026-10-15T04:31:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:33.494+00:00 2026-10-15T04:31:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:34.498+00:00 2026-10-15T04:31:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:35.503+00:00 2026-10-15T04:31:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:36.512+00:00 2026-10-15T04:31:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:37.517+00:00 2026-10-15T04:31:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:38.518+00:00 2026-10-15T04:31:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:39.523+00:00 2026-10-15T04:31:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:40.528+00:00 2026-10-15T04:31:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:41.532+00:00 2026-10-15T04:31:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:42.537+00:00 2026-10-15T04:31:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:43.543+00:00 2026-10-15T04:31:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:44.547+00:00 2026-10-15T04:31:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:45.561+00:00 2026-10-15T04:31:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:46.559+00:00 2026-10-15T04:31:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:47.570+00:00 2026-10-15T04:31:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:48.578+00:00 2026-10-15T04:31:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:49.574+00:00 2026-10-15T04:31:49Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:49.574+00:00 2026-10-15T04:31:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:50.579+00:00 2026-10-15T04:31:50Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:50.579+00:00 2026-10-15T04:31:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:51.585+00:00 2026-10-15T04:31:51Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:51.585+00:00 2026-10-15T04:31:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:52.592+00:00 2026-10-15T04:31:52Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:52.592+00:00 2026-10-15T04:31:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:53.594+00:00 2026-10-15T04:31:53Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:53.594+00:00 2026-10-15T04:31:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:54.612+00:00 2026-10-15T04:31:54Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:54.612+00:00 2026-10-15T04:31:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:55.605+00:00 2026-10-15T04:31:55Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:55.605+00:00 2026-10-15T04:31:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T04:31:56.610+00:00 2026-10-15T04:31:56Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T04:31:56.610+00:00 2026-10-15T04:31:56Z open_rl:trainer_ready:Qwen/Qwen3-8B diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log b/runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log deleted file mode 100644 index f36aa39c..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T04:28:48.737+00:00 2026-10-15T04:28:48.736460900Z {"level":"info","ts":"2026-10-15T04:28:48Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"9810872f-787e-462c-b16a-dceac5ec5ffa","claim":"claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} -2026-10-15T04:29:47.974+00:00 2026-10-15T04:29:47.973626989Z {"level":"info","ts":"2026-10-15T04:29:47Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"a0e0e96a-42aa-4961-903d-5120a533b95f","claim":"claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json deleted file mode 100644 index 90585ed3..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/timeline.json +++ /dev/null @@ -1,267 +0,0 @@ -{ - "case": "D-8b-h100-cached", - "collected": "2026-10-15T04:32:14.200+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 04:28:47.756000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen3-8B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 04:28:47.756000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 04:29:39.925000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "db28c229-9bd6-4845-bd41-edb2e3018375" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 04:29:42.710000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 04:29:45.623000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 04:29:47.732000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 04:29:47.733000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 04:29:49.654000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 04:29:49.657000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 04:31:50.315000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4\n\nQuestion: What is " - }, - "client.end": { - "ts": "2026-10-15 04:31:50.316000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen3-8b-0-sampler", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "pod": "orw-lora-qwen-qwen3-8b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 04:29:47+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 04:29:47.960000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 04:29:47.960000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 04:29:47.960000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 04:29:52+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 04:29:48+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 04:29:48+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 04:29:52+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 04:29:52+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 04:29:48+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 04:29:52+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 04:29:48+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 04:29:51+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "sampler.event_Created": { - "ts": "2026-10-15 04:29:51+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 04:29:52+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 04:29:59.120383+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 04:30:16.823040+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:16 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 04:30:17.937691+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:17 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 04:30:18.941839+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:18 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 04:30:31.464946+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:31 [default_loader.py:430] Loading weights took 12.53 seconds", - "self_reported_s": 12.53 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 04:30:32.103943+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:30:32 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.439416 seconds", - "self_reported_s": 14.439416 - }, - "sampler.compile_done": { - "ts": "2026-10-15 04:31:08.896773+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:08 [monitor.py:53] torch.compile took 35.76 s in total", - "self_reported_s": 35.76 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 04:31:29.315428+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:29 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", - "self_reported_s": 16.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 04:31:29.941703+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:29 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 04:31:48.438271+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 04:31:48 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.33 s", - "self_reported_s": 76.33 - }, - "sampler.ready_key": { - "ts": "2026-10-15 04:31:49+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" - }, - "trainer": { - "workload": "lora-qwen-qwen3-8b-0-trainer", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "pod": "orw-lora-qwen-qwen3-8b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 04:28:48+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 04:28:48.508000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 04:28:48.508000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 04:28:48.508000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 04:28:53+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 04:28:48+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 04:28:48+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 04:28:53+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 04:28:53+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 04:28:48+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 04:28:52+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 04:28:48+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 04:28:52+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "trainer.event_Created": { - "ts": "2026-10-15 04:28:52+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 04:28:52+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 04:28:57.210385+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 04:28:57.431621+00:00", - "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 04:29:37.342144+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 04:29:37.343864+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 04:29:37+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:0ae950e226f5400262fc3b207f6bde8d5c02e7d02fb15f6ae1ad9e3d55e034d0" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt deleted file mode 100644 index 777815ab..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T04:28:43.565+00:00 diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt b/runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt deleted file mode 100644 index 0d162c07..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13 \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json b/runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json deleted file mode 100644 index 3f3cb447..00000000 --- a/runs/2026-10-15_cold_start/D-8b-h100-cached/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:47Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038592335647008", - "uid": "aac5fdd5-1826-49c7-8392-5165024b0a69" - }, - "spec": { - "accelerator": { - "memory": "29Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "name": "worker", - "resources": { - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "64aee2d3-084c-46a4-9c77-df7f6442e580", - "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T04:29:52Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-aac5fdd5", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "29Gi", - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038533448655004", - "uid": "cde28d2b-7497-4559-ae79-2c350558ddd1" - }, - "spec": { - "accelerator": { - "memory": "20Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13", - "name": "worker", - "resources": { - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "6521caef-db14-4d7b-8477-e644e69919ee", - "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T04:28:53Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-cde28d2b", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "20Gi", - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log deleted file mode 100644 index f8ffab7c..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold.run.log +++ /dev/null @@ -1,35 +0,0 @@ -watching; recorders [12362, 12363, 12365, 12367, 12369, 12364, 12366, 12368] write into runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold -collected 2 worker log(s) into runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.9 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 172.0 | -| container create | trainer.event_Pulled | trainer.container_started | 1.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.2 | -| weights load | trainer.load_start | trainer.load_done | 15.0 | -| ready key | trainer.load_done | trainer.ready_key | 0.7 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 197.1 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.1 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 186.0 | -| container create | sampler.event_Pulled | sampler.container_started | 1.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 23.2 | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 15.2 | -| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.7 | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.4 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.5 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.7 | -| ready key | sampler.engine_init_done | sampler.ready_key | 1.2 | -| client: first sample blocked | client.sample.start | client.sample.done | 289.2 | - diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log deleted file mode 100644 index 72baa052..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/api-server.log +++ /dev/null @@ -1,75 +0,0 @@ -2026-10-15T04:47:50.759+00:00 2026-10-15T04:47:50.747267827Z INFO: 10.32.3.1:50030 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:47:51.373+00:00 2026-10-15T04:47:51.375072109Z INFO: 127.0.0.1:58948 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T04:47:51.625+00:00 2026-10-15T04:47:51.626679728Z INFO: 127.0.0.1:58962 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T04:47:51.706+00:00 2026-10-15T04:47:51.698589265Z INFO: 127.0.0.1:58962 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T04:47:51.964+00:00 2026-10-15T04:47:51.960105348Z [API_SERVER] enqueued op=create_model request_id=c34fd223-1dd8-443e-850e-68d16b0d897e model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:47:51.965+00:00 2026-10-15T04:47:51.961209781Z INFO: 127.0.0.1:58976 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T04:48:00.748+00:00 2026-10-15T04:48:00.747101104Z INFO: 10.32.3.1:44106 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:48:01.881+00:00 2026-10-15T04:48:01.882432111Z INFO: 127.0.0.1:39078 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:48:10.748+00:00 2026-10-15T04:48:10.746655455Z INFO: 10.32.3.1:43406 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:48:20.762+00:00 2026-10-15T04:48:20.747630744Z INFO: 10.32.3.1:50344 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:48:30.750+00:00 2026-10-15T04:48:30.747452523Z INFO: 10.32.3.1:38480 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:48:40.748+00:00 2026-10-15T04:48:40.747887168Z INFO: 10.32.3.1:43688 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:48:42.107+00:00 2026-10-15T04:48:42.108647369Z INFO: 127.0.0.1:44312 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:48:50.747+00:00 2026-10-15T04:48:50.746753362Z INFO: 10.32.3.1:56992 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:49:00.759+00:00 2026-10-15T04:49:00.748474238Z INFO: 10.32.3.1:51752 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:49:10.753+00:00 2026-10-15T04:49:10.747254442Z INFO: 10.32.3.1:60438 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:49:20.748+00:00 2026-10-15T04:49:20.747384062Z INFO: 10.32.3.1:55404 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:49:30.744+00:00 2026-10-15T04:49:30.748502393Z INFO: 10.32.3.1:51894 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:49:32.332+00:00 2026-10-15T04:49:32.338576070Z INFO: 127.0.0.1:50908 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:49:40.737+00:00 2026-10-15T04:49:40.746618093Z INFO: 10.32.3.1:59074 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:49:50.742+00:00 2026-10-15T04:49:50.747114846Z INFO: 10.32.3.1:41452 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:50:00.737+00:00 2026-10-15T04:50:00.747761065Z INFO: 10.32.3.1:58260 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:50:10.736+00:00 2026-10-15T04:50:10.747653348Z INFO: 10.32.3.1:50724 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:50:12.511+00:00 2026-10-15T04:50:12.524698371Z INFO: 127.0.0.1:57438 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:50:20.733+00:00 2026-10-15T04:50:20.746488844Z INFO: 10.32.3.1:37876 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:50:30.735+00:00 2026-10-15T04:50:30.747506581Z INFO: 10.32.3.1:46886 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:50:40.734+00:00 2026-10-15T04:50:40.747651246Z INFO: 10.32.3.1:47538 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:50:50.734+00:00 2026-10-15T04:50:50.747642546Z INFO: 10.32.3.1:42666 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:51:00.735+00:00 2026-10-15T04:51:00.747457778Z INFO: 10.32.3.1:51346 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:51:02.734+00:00 2026-10-15T04:51:02.748492339Z INFO: 127.0.0.1:33774 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:51:08.372+00:00 2026-10-15T04:51:08.384894234Z INFO: 127.0.0.1:33804 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T04:51:10.734+00:00 2026-10-15T04:51:10.747373366Z INFO: 10.32.3.1:44790 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:51:11.018+00:00 2026-10-15T04:51:11.033251000Z [API_SERVER] enqueued op=forward_backward request_id=82deca49-4ee4-496f-825c-c12a291fb417 model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:51:11.018+00:00 2026-10-15T04:51:11.033591274Z INFO: 127.0.0.1:33804 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T04:51:14.259+00:00 2026-10-15T04:51:14.268225387Z [API_SERVER] enqueued op=optim_step request_id=f6a0aff8-ac17-496d-a35f-82a1866beb3c model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:51:14.259+00:00 2026-10-15T04:51:14.269572118Z INFO: 127.0.0.1:33804 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T04:51:15.185+00:00 2026-10-15T04:51:15.196153109Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=29042980-72d1-47bc-b560-caa782c86947 model_id=c34fd223-1dd8-443e-850e-68d16b0d897e active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:51:15.185+00:00 2026-10-15T04:51:15.196730876Z INFO: 127.0.0.1:33804 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T04:51:15.847+00:00 2026-10-15T04:51:15.861994037Z INFO: 127.0.0.1:50962 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T04:51:20.733+00:00 2026-10-15T04:51:20.747509561Z INFO: 10.32.3.1:36522 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:51:30.734+00:00 2026-10-15T04:51:30.747769712Z INFO: 10.32.3.1:49072 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:51:40.735+00:00 2026-10-15T04:51:40.747089500Z INFO: 10.32.3.1:42936 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:51:50.807+00:00 2026-10-15T04:51:50.747657379Z INFO: 10.32.3.1:59096 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:52:00.735+00:00 2026-10-15T04:52:00.746837023Z INFO: 10.32.3.1:44932 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:52:02.930+00:00 2026-10-15T04:52:02.943641729Z INFO: 127.0.0.1:60640 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:52:10.736+00:00 2026-10-15T04:52:10.747747798Z INFO: 10.32.3.1:44974 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:52:20.736+00:00 2026-10-15T04:52:20.746678567Z INFO: 10.32.3.1:35664 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:52:30.734+00:00 2026-10-15T04:52:30.747126329Z INFO: 10.32.3.1:34208 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:52:40.735+00:00 2026-10-15T04:52:40.747196297Z INFO: 10.32.3.1:44174 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:52:50.734+00:00 2026-10-15T04:52:50.747115733Z INFO: 10.32.3.1:54892 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:52:51.758+00:00 2026-10-15T04:52:51.774099202Z INFO: 127.0.0.1:40930 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T04:52:53.102+00:00 2026-10-15T04:52:53.117722316Z INFO: 127.0.0.1:54688 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:53:00.733+00:00 2026-10-15T04:53:00.747018328Z INFO: 10.32.3.1:34592 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:53:10.739+00:00 2026-10-15T04:53:10.747952567Z INFO: 10.32.3.1:34466 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:53:20.735+00:00 2026-10-15T04:53:20.747402168Z INFO: 10.32.3.1:43034 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:53:30.735+00:00 2026-10-15T04:53:30.747874323Z INFO: 10.32.3.1:44336 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:53:40.734+00:00 2026-10-15T04:53:40.747391997Z INFO: 10.32.3.1:50648 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:53:43.288+00:00 2026-10-15T04:53:43.301447514Z INFO: 127.0.0.1:35782 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:53:50.735+00:00 2026-10-15T04:53:50.748058844Z INFO: 10.32.3.1:44446 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:54:00.741+00:00 2026-10-15T04:54:00.747477817Z INFO: 10.32.3.1:60436 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:54:10.733+00:00 2026-10-15T04:54:10.747726850Z INFO: 10.32.3.1:59106 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:54:20.768+00:00 2026-10-15T04:54:20.747514561Z INFO: 10.32.3.1:37768 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:54:23.560+00:00 2026-10-15T04:54:23.574498792Z INFO: 127.0.0.1:42366 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:54:30.734+00:00 2026-10-15T04:54:30.747280852Z INFO: 10.32.3.1:47062 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:54:40.733+00:00 2026-10-15T04:54:40.746421659Z INFO: 10.32.3.1:53944 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:54:50.739+00:00 2026-10-15T04:54:50.747856422Z INFO: 10.32.3.1:50702 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:55:00.789+00:00 2026-10-15T04:55:00.747454361Z INFO: 10.32.3.1:48938 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:55:10.733+00:00 2026-10-15T04:55:10.747058641Z INFO: 10.32.3.1:48650 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:55:20.734+00:00 2026-10-15T04:55:20.747467882Z INFO: 10.32.3.1:54104 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:55:23.758+00:00 2026-10-15T04:55:23.773354564Z INFO: 127.0.0.1:59436 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:55:30.733+00:00 2026-10-15T04:55:30.747006683Z INFO: 10.32.3.1:51618 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:55:40.733+00:00 2026-10-15T04:55:40.747561980Z INFO: 10.32.3.1:60968 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:55:50.733+00:00 2026-10-15T04:55:50.747513210Z INFO: 10.32.3.1:41264 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:56:00.733+00:00 2026-10-15T04:56:00.747491999Z INFO: 10.32.3.1:32776 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:56:10.733+00:00 2026-10-15T04:56:10.746647279Z INFO: 10.32.3.1:51500 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl deleted file mode 100644 index b94dfe70..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T04:47:51.110+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T04:47:51.111+00:00", "t": 0.0, "phase": "create_model.start"} -{"ts": "2026-10-15T04:51:08.186+00:00", "t": 197.076, "phase": "create_model.done", "model_id": "c34fd223-1dd8-443e-850e-68d16b0d897e"} -{"ts": "2026-10-15T04:51:10.937+00:00", "t": 199.827, "phase": "forward_backward.start"} -{"ts": "2026-10-15T04:51:14.170+00:00", "t": 203.059, "phase": "forward_backward.done"} -{"ts": "2026-10-15T04:51:15.072+00:00", "t": 203.961, "phase": "optim_step.done"} -{"ts": "2026-10-15T04:51:15.072+00:00", "t": 203.962, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T04:51:15.661+00:00", "t": 204.551, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T04:51:15.662+00:00", "t": 204.551, "phase": "sample.start"} -{"ts": "2026-10-15T04:56:04.835+00:00", "t": 493.725, "phase": "sample.done", "text": " 4"} -{"ts": "2026-10-15T04:56:04.838+00:00", "t": 493.728, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log deleted file mode 100644 index a1f626f6..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.000s create_model.start -[cold-start] 197.076s create_model.done {'model_id': 'c34fd223-1dd8-443e-850e-68d16b0d897e'} -[cold-start] 199.827s forward_backward.start -[cold-start] 203.059s forward_backward.done -[cold-start] 203.961s optim_step.done -[cold-start] 203.962s save_weights_and_get_sampling_client.start -[cold-start] 204.551s save_weights_and_get_sampling_client.done -[cold-start] 204.551s sample.start -[cold-start] 493.725s sample.done {'text': ' 4'} -[cold-start] 493.728s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt deleted file mode 100644 index 82fe3f4d..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T04:56:05.389+00:00 diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json deleted file mode 100644 index 7497a20b..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/events.json +++ /dev/null @@ -1,1311 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:10Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:10Z", - "name": "cold-start-fio.18de9a0ef295773d", - "namespace": "openrl-system", - "resourceVersion": "1792039090760383011", - "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5be63862", - "namespace": "openrl-system", - "resourceVersion": "1792039092515007000", - "uid": "355b59b9-3661-499d-8640-3f40c3c89375" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5ecfa51a", - "namespace": "openrl-system", - "resourceVersion": "1792039092561663021", - "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f61ddd1e4", - "namespace": "openrl-system", - "resourceVersion": "1792039092605935011", - "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:40:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:40:41Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:40:41Z", - "name": "cold-start-fio.18de9a32081cf908", - "namespace": "openrl-system", - "resourceVersion": "1792039241446287009", - "uid": "14a62e42-13ba-4526-a04d-a934702e1163" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:18Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:18Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:18Z", - "name": "cold-start-fio.18de9a3a9867ce9f", - "namespace": "openrl-system", - "resourceVersion": "1792039278208207001", - "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad2bcac91", - "namespace": "openrl-system", - "resourceVersion": "1792039279188751014", - "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad378cd74", - "namespace": "openrl-system", - "resourceVersion": "1792039279219679006", - "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad5aed221", - "namespace": "openrl-system", - "resourceVersion": "1792039279235135013", - "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:11Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:11Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:11Z", - "name": "cold-start-fio.18de9a70cd2c6a68", - "namespace": "openrl-system", - "resourceVersion": "1792039511048479001", - "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "resourceVersion": "1791941114537823021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Stopping container api-server", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", - "namespace": "openrl-system", - "resourceVersion": "1792039545418735013", - "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "namespace": "openrl-system", - "resourceVersion": "1792039545300351007", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", - "namespace": "openrl-system", - "resourceVersion": "1792039545387567001", - "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" - }, - "reason": "SuccessfulDelete", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546048591003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", - "namespace": "openrl-system", - "resourceVersion": "1792039546102351012", - "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:49Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:49Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", - "namespace": "openrl-system", - "resourceVersion": "1792039549712335019", - "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", - "namespace": "openrl-system", - "resourceVersion": "1792039558721215017", - "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", - "namespace": "openrl-system", - "resourceVersion": "1792039558783199013", - "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", - "namespace": "openrl-system", - "resourceVersion": "1792039558862303009", - "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "namespace": "openrl-system", - "resourceVersion": "1792039545940671018", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", - "namespace": "openrl-system", - "resourceVersion": "1792039546068975009", - "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" - }, - "reason": "SuccessfulCreate", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545232655018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78c9668cf7", - "namespace": "openrl-system", - "resourceVersion": "1792039545327119005", - "uid": "7b1433df-3524-478b-aba2-8979950e245e" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545464319018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78ef8b9e1b", - "namespace": "openrl-system", - "resourceVersion": "1792039545956079020", - "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:15Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875490623001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:15Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", - "namespace": "openrl-system", - "resourceVersion": "1792039875609279013", - "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:19Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:19Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", - "namespace": "openrl-system", - "resourceVersion": "1792039879126319008", - "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", - "namespace": "openrl-system", - "resourceVersion": "1792040061645247020", - "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", - "namespace": "openrl-system", - "resourceVersion": "1792040061849391002", - "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", - "namespace": "openrl-system", - "resourceVersion": "1792040062252079023", - "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:52Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672220015012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:52Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", - "namespace": "openrl-system", - "resourceVersion": "1792039672350591019", - "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:56Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:56Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:56Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", - "namespace": "openrl-system", - "resourceVersion": "1792039676166239015", - "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", - "namespace": "openrl-system", - "resourceVersion": "1792039844457167000", - "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", - "namespace": "openrl-system", - "resourceVersion": "1792039844632591021", - "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:45Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:45Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", - "namespace": "openrl-system", - "resourceVersion": "1792039845170959016", - "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", - "namespace": "openrl-system", - "resourceVersion": "1792038835732623016", - "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", - "namespace": "openrl-system", - "resourceVersion": "1792038835415695012", - "uid": "918abc9a-1f24-4767-8928-b202482f11ed" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt deleted file mode 100644 index e82879b1..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt +++ /dev/null @@ -1,120 +0,0 @@ -2026-10-15T04:54:45.242588287Z INFO 10-15 04:54:45 [model.py:692] Resolved architecture: Qwen2ForCausalLM -2026-10-15T04:54:45.277423658Z INFO 10-15 04:54:45 [model.py:2030] Using max model len 8192 -2026-10-15T04:54:45.277455256Z INFO 10-15 04:54:45 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. -2026-10-15T04:54:45.327374906Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T04:54:45.587816185Z INFO 10-15 04:54:45 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T04:54:48.964429054Z WARNING 10-15 04:54:48 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T04:55:00.403987147Z (EngineCore pid=161) INFO 10-15 04:55:00 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T04:55:01.212591231Z (EngineCore pid=161) INFO 10-15 04:55:01 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_14e101098568499f8d634b8f3ba36063 backend=nccl -2026-10-15T04:55:01.219901527Z (EngineCore pid=161) INFO 10-15 04:55:01 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T04:55:01.219928991Z (EngineCore pid=161) INFO 10-15 04:55:01 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T04:55:02.093177635Z (EngineCore pid=161) INFO 10-15 04:55:02 [model_runner.py:396] Loading model from scratch... -2026-10-15T04:55:02.961631574Z (EngineCore pid=161) INFO 10-15 04:55:02 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T04:55:02.962148325Z (EngineCore pid=161) INFO 10-15 04:55:02 [flash_attn.py:1116] Using FlashAttention version 2 -2026-10-15T04:55:03.077952928Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T04:55:03.078372599Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T04:55:03.438996437Z (EngineCore pid=161) INFO 10-15 04:55:03 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.61 GiB. -2026-10-15T04:55:03.439031206Z (EngineCore pid=161) INFO 10-15 04:55:03 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T04:55:03.440364205Z (EngineCore pid=161) -Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T04:50:52.100407463Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T04:50:52.100446712Z -2026-10-15T04:50:52.221389461Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... -2026-10-15T04:50:52.374261114Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T04:50:53.898909828Z [prefetch] reading 1 file(s), 0.92 GiB, from nfs into the page cache with 8 threads -2026-10-15T04:51:06.274074801Z [prefetch] done: 0.92 GiB in 12.4 s (76 MiB/s) -2026-10-15T04:51:07.154043242Z -Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json deleted file mode 100644 index 7e1e5910..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "generateName": "open-rl-api-server-6d86b498f5-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "6d86b498f5", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - } - ], - "resourceVersion": "1792039570790271003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2ns79", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:59Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:45:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.11", - "podIPs": [ - { - "ip": "10.32.3.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:45:46Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "uid": "0752a694-3e75-44aa-9348-ae1c3f3fff13" - } - ], - "resourceVersion": "1792040062433135001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "ed5c4cfd-e94a-4a7f-a2a6-62ba293d8d04" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-7xf7q", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-7xf7q", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:54:22Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:51:15Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:54:22Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:54:22Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:51:15Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "25Gi" - }, - "containerID": "containerd://9e18e8280c01871067b298e040014667b0807f38eaf685466bc2c46ce2b4a65d", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:54:22Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-7xf7q", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.0.20", - "podIPs": [ - { - "ip": "10.32.0.20" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:51:15Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "uid": "4bef5a35-3bf8-4444-9043-47f86b502e3e" - } - ], - "resourceVersion": "1792039845726895012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "0772feaa-10d8-4243-ac6a-4b4ee0615042" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-hrg2h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-hrg2h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:50:45Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:47:52Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:50:45Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:50:45Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:47:52Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "21Gi" - }, - "containerID": "containerd://91dafb207ae62faa4acac97ecaac86ef915d83f785b30550632f947d515e7601", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:50:45Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-hrg2h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.13.26", - "podIPs": [ - { - "ip": "10.32.13.26" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:47:52Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log deleted file mode 100644 index 51432475..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/redis-ready.log +++ /dev/null @@ -1,309 +0,0 @@ -2026-10-15T04:51:08.025+00:00 2026-10-15T04:51:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:09.029+00:00 2026-10-15T04:51:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:10.034+00:00 2026-10-15T04:51:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:11.039+00:00 2026-10-15T04:51:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:12.044+00:00 2026-10-15T04:51:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:13.052+00:00 2026-10-15T04:51:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:14.055+00:00 2026-10-15T04:51:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:15.071+00:00 2026-10-15T04:51:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:16.065+00:00 2026-10-15T04:51:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:17.071+00:00 2026-10-15T04:51:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:18.081+00:00 2026-10-15T04:51:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:19.081+00:00 2026-10-15T04:51:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:20.088+00:00 2026-10-15T04:51:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:21.105+00:00 2026-10-15T04:51:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:22.098+00:00 2026-10-15T04:51:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:23.103+00:00 2026-10-15T04:51:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:24.108+00:00 2026-10-15T04:51:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:25.113+00:00 2026-10-15T04:51:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:26.119+00:00 2026-10-15T04:51:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:27.123+00:00 2026-10-15T04:51:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:28.131+00:00 2026-10-15T04:51:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:29.133+00:00 2026-10-15T04:51:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:30.139+00:00 2026-10-15T04:51:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:31.144+00:00 2026-10-15T04:51:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:32.150+00:00 2026-10-15T04:51:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:33.155+00:00 2026-10-15T04:51:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:34.159+00:00 2026-10-15T04:51:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:35.164+00:00 2026-10-15T04:51:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:36.175+00:00 2026-10-15T04:51:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:37.175+00:00 2026-10-15T04:51:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:38.179+00:00 2026-10-15T04:51:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:39.184+00:00 2026-10-15T04:51:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:40.189+00:00 2026-10-15T04:51:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:41.313+00:00 2026-10-15T04:51:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:42.202+00:00 2026-10-15T04:51:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:43.204+00:00 2026-10-15T04:51:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:44.209+00:00 2026-10-15T04:51:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:45.227+00:00 2026-10-15T04:51:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:46.219+00:00 2026-10-15T04:51:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:47.224+00:00 2026-10-15T04:51:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:48.229+00:00 2026-10-15T04:51:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:49.234+00:00 2026-10-15T04:51:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:50.240+00:00 2026-10-15T04:51:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:51.348+00:00 2026-10-15T04:51:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:52.251+00:00 2026-10-15T04:51:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:53.255+00:00 2026-10-15T04:51:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:54.261+00:00 2026-10-15T04:51:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:55.266+00:00 2026-10-15T04:51:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:56.273+00:00 2026-10-15T04:51:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:57.278+00:00 2026-10-15T04:51:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:58.284+00:00 2026-10-15T04:51:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:51:59.351+00:00 2026-10-15T04:51:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:00.293+00:00 2026-10-15T04:52:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:01.309+00:00 2026-10-15T04:52:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:02.312+00:00 2026-10-15T04:52:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:03.310+00:00 2026-10-15T04:52:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:04.313+00:00 2026-10-15T04:52:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:05.319+00:00 2026-10-15T04:52:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:06.323+00:00 2026-10-15T04:52:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:07.328+00:00 2026-10-15T04:52:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:08.333+00:00 2026-10-15T04:52:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:09.338+00:00 2026-10-15T04:52:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:10.342+00:00 2026-10-15T04:52:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:11.347+00:00 2026-10-15T04:52:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:12.352+00:00 2026-10-15T04:52:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:13.358+00:00 2026-10-15T04:52:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:14.362+00:00 2026-10-15T04:52:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:15.421+00:00 2026-10-15T04:52:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:16.374+00:00 2026-10-15T04:52:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:17.420+00:00 2026-10-15T04:52:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:18.381+00:00 2026-10-15T04:52:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:19.386+00:00 2026-10-15T04:52:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:20.391+00:00 2026-10-15T04:52:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:21.397+00:00 2026-10-15T04:52:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:22.416+00:00 2026-10-15T04:52:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:23.406+00:00 2026-10-15T04:52:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:24.412+00:00 2026-10-15T04:52:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:25.416+00:00 2026-10-15T04:52:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:26.422+00:00 2026-10-15T04:52:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:27.609+00:00 2026-10-15T04:52:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:28.431+00:00 2026-10-15T04:52:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:29.442+00:00 2026-10-15T04:52:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:30.443+00:00 2026-10-15T04:52:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:31.449+00:00 2026-10-15T04:52:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:32.454+00:00 2026-10-15T04:52:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:33.459+00:00 2026-10-15T04:52:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:34.465+00:00 2026-10-15T04:52:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:35.472+00:00 2026-10-15T04:52:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:36.474+00:00 2026-10-15T04:52:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:37.479+00:00 2026-10-15T04:52:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:38.484+00:00 2026-10-15T04:52:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:39.501+00:00 2026-10-15T04:52:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:40.495+00:00 2026-10-15T04:52:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:41.502+00:00 2026-10-15T04:52:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:42.508+00:00 2026-10-15T04:52:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:43.510+00:00 2026-10-15T04:52:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:44.515+00:00 2026-10-15T04:52:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:45.520+00:00 2026-10-15T04:52:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:46.537+00:00 2026-10-15T04:52:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:47.531+00:00 2026-10-15T04:52:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:48.536+00:00 2026-10-15T04:52:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:49.541+00:00 2026-10-15T04:52:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:50.546+00:00 2026-10-15T04:52:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:51.551+00:00 2026-10-15T04:52:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:52.561+00:00 2026-10-15T04:52:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:53.561+00:00 2026-10-15T04:52:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:54.566+00:00 2026-10-15T04:52:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:55.570+00:00 2026-10-15T04:52:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:56.577+00:00 2026-10-15T04:52:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:57.581+00:00 2026-10-15T04:52:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:58.585+00:00 2026-10-15T04:52:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:52:59.590+00:00 2026-10-15T04:52:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:00.599+00:00 2026-10-15T04:53:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:01.601+00:00 2026-10-15T04:53:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:02.606+00:00 2026-10-15T04:53:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:03.610+00:00 2026-10-15T04:53:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:04.616+00:00 2026-10-15T04:53:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:05.622+00:00 2026-10-15T04:53:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:06.626+00:00 2026-10-15T04:53:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:07.631+00:00 2026-10-15T04:53:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:08.636+00:00 2026-10-15T04:53:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:09.643+00:00 2026-10-15T04:53:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:10.648+00:00 2026-10-15T04:53:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:11.652+00:00 2026-10-15T04:53:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:12.657+00:00 2026-10-15T04:53:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:13.663+00:00 2026-10-15T04:53:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:14.667+00:00 2026-10-15T04:53:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:15.672+00:00 2026-10-15T04:53:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:16.678+00:00 2026-10-15T04:53:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:17.683+00:00 2026-10-15T04:53:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:18.686+00:00 2026-10-15T04:53:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:19.691+00:00 2026-10-15T04:53:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:20.697+00:00 2026-10-15T04:53:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:21.702+00:00 2026-10-15T04:53:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:22.707+00:00 2026-10-15T04:53:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:23.713+00:00 2026-10-15T04:53:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:24.717+00:00 2026-10-15T04:53:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:25.722+00:00 2026-10-15T04:53:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:26.727+00:00 2026-10-15T04:53:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:27.733+00:00 2026-10-15T04:53:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:28.743+00:00 2026-10-15T04:53:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:29.743+00:00 2026-10-15T04:53:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:30.748+00:00 2026-10-15T04:53:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:31.760+00:00 2026-10-15T04:53:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:32.758+00:00 2026-10-15T04:53:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:33.764+00:00 2026-10-15T04:53:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:34.770+00:00 2026-10-15T04:53:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:35.779+00:00 2026-10-15T04:53:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:36.780+00:00 2026-10-15T04:53:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:37.786+00:00 2026-10-15T04:53:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:38.791+00:00 2026-10-15T04:53:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:39.797+00:00 2026-10-15T04:53:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:40.801+00:00 2026-10-15T04:53:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:41.807+00:00 2026-10-15T04:53:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:42.811+00:00 2026-10-15T04:53:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:43.816+00:00 2026-10-15T04:53:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:44.821+00:00 2026-10-15T04:53:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:45.826+00:00 2026-10-15T04:53:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:46.830+00:00 2026-10-15T04:53:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:47.837+00:00 2026-10-15T04:53:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:48.845+00:00 2026-10-15T04:53:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:49.847+00:00 2026-10-15T04:53:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:50.854+00:00 2026-10-15T04:53:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:51.856+00:00 2026-10-15T04:53:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:52.862+00:00 2026-10-15T04:53:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:53.867+00:00 2026-10-15T04:53:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:54.871+00:00 2026-10-15T04:53:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:55.877+00:00 2026-10-15T04:53:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:56.881+00:00 2026-10-15T04:53:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:57.886+00:00 2026-10-15T04:53:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:58.891+00:00 2026-10-15T04:53:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:53:59.898+00:00 2026-10-15T04:53:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:00.901+00:00 2026-10-15T04:54:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:01.907+00:00 2026-10-15T04:54:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:02.910+00:00 2026-10-15T04:54:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:03.923+00:00 2026-10-15T04:54:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:04.928+00:00 2026-10-15T04:54:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:05.930+00:00 2026-10-15T04:54:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:07.036+00:00 2026-10-15T04:54:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:07.939+00:00 2026-10-15T04:54:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:08.945+00:00 2026-10-15T04:54:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:09.950+00:00 2026-10-15T04:54:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:10.955+00:00 2026-10-15T04:54:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:11.989+00:00 2026-10-15T04:54:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:12.973+00:00 2026-10-15T04:54:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:13.973+00:00 2026-10-15T04:54:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:14.976+00:00 2026-10-15T04:54:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:15.981+00:00 2026-10-15T04:54:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:16.987+00:00 2026-10-15T04:54:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:17.991+00:00 2026-10-15T04:54:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:19.054+00:00 2026-10-15T04:54:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:20.002+00:00 2026-10-15T04:54:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:21.008+00:00 2026-10-15T04:54:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:22.074+00:00 2026-10-15T04:54:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:23.040+00:00 2026-10-15T04:54:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:24.021+00:00 2026-10-15T04:54:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:25.025+00:00 2026-10-15T04:54:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:26.029+00:00 2026-10-15T04:54:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:27.035+00:00 2026-10-15T04:54:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:28.050+00:00 2026-10-15T04:54:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:29.096+00:00 2026-10-15T04:54:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:30.056+00:00 2026-10-15T04:54:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:31.058+00:00 2026-10-15T04:54:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:32.065+00:00 2026-10-15T04:54:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:33.067+00:00 2026-10-15T04:54:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:34.072+00:00 2026-10-15T04:54:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:35.077+00:00 2026-10-15T04:54:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:36.085+00:00 2026-10-15T04:54:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:37.089+00:00 2026-10-15T04:54:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:38.093+00:00 2026-10-15T04:54:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:39.099+00:00 2026-10-15T04:54:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:40.106+00:00 2026-10-15T04:54:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:41.110+00:00 2026-10-15T04:54:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:42.114+00:00 2026-10-15T04:54:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:43.119+00:00 2026-10-15T04:54:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:44.202+00:00 2026-10-15T04:54:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:45.129+00:00 2026-10-15T04:54:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:46.133+00:00 2026-10-15T04:54:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:47.138+00:00 2026-10-15T04:54:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:48.142+00:00 2026-10-15T04:54:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:49.148+00:00 2026-10-15T04:54:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:50.152+00:00 2026-10-15T04:54:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:51.162+00:00 2026-10-15T04:54:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:52.163+00:00 2026-10-15T04:54:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:53.174+00:00 2026-10-15T04:54:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:54.172+00:00 2026-10-15T04:54:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:55.190+00:00 2026-10-15T04:54:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:56.182+00:00 2026-10-15T04:54:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:57.187+00:00 2026-10-15T04:54:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:58.192+00:00 2026-10-15T04:54:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:54:59.196+00:00 2026-10-15T04:54:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:00.202+00:00 2026-10-15T04:55:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:01.207+00:00 2026-10-15T04:55:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:02.213+00:00 2026-10-15T04:55:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:03.216+00:00 2026-10-15T04:55:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:04.221+00:00 2026-10-15T04:55:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:05.239+00:00 2026-10-15T04:55:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:06.232+00:00 2026-10-15T04:55:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:07.237+00:00 2026-10-15T04:55:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:08.243+00:00 2026-10-15T04:55:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:09.248+00:00 2026-10-15T04:55:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:10.264+00:00 2026-10-15T04:55:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:11.259+00:00 2026-10-15T04:55:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:12.263+00:00 2026-10-15T04:55:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:13.268+00:00 2026-10-15T04:55:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:14.321+00:00 2026-10-15T04:55:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:15.279+00:00 2026-10-15T04:55:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:16.284+00:00 2026-10-15T04:55:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:17.289+00:00 2026-10-15T04:55:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:18.294+00:00 2026-10-15T04:55:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:19.299+00:00 2026-10-15T04:55:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:20.304+00:00 2026-10-15T04:55:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:21.309+00:00 2026-10-15T04:55:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:22.322+00:00 2026-10-15T04:55:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:23.324+00:00 2026-10-15T04:55:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:24.323+00:00 2026-10-15T04:55:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:25.328+00:00 2026-10-15T04:55:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:26.334+00:00 2026-10-15T04:55:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:27.338+00:00 2026-10-15T04:55:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:28.403+00:00 2026-10-15T04:55:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:29.351+00:00 2026-10-15T04:55:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:30.355+00:00 2026-10-15T04:55:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:31.358+00:00 2026-10-15T04:55:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:32.363+00:00 2026-10-15T04:55:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:33.376+00:00 2026-10-15T04:55:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:34.378+00:00 2026-10-15T04:55:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:35.378+00:00 2026-10-15T04:55:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:36.383+00:00 2026-10-15T04:55:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:37.389+00:00 2026-10-15T04:55:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:38.397+00:00 2026-10-15T04:55:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:39.413+00:00 2026-10-15T04:55:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:40.403+00:00 2026-10-15T04:55:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:41.408+00:00 2026-10-15T04:55:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:42.416+00:00 2026-10-15T04:55:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:43.418+00:00 2026-10-15T04:55:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:44.424+00:00 2026-10-15T04:55:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:45.428+00:00 2026-10-15T04:55:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:46.433+00:00 2026-10-15T04:55:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:47.442+00:00 2026-10-15T04:55:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:48.443+00:00 2026-10-15T04:55:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:49.458+00:00 2026-10-15T04:55:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:50.453+00:00 2026-10-15T04:55:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:51.459+00:00 2026-10-15T04:55:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:52.463+00:00 2026-10-15T04:55:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:53.472+00:00 2026-10-15T04:55:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:54.474+00:00 2026-10-15T04:55:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:55.479+00:00 2026-10-15T04:55:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:56.483+00:00 2026-10-15T04:55:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:57.488+00:00 2026-10-15T04:55:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:58.492+00:00 2026-10-15T04:55:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:55:59.499+00:00 2026-10-15T04:55:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:00.503+00:00 2026-10-15T04:56:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:01.508+00:00 2026-10-15T04:56:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:02.514+00:00 2026-10-15T04:56:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:03.518+00:00 2026-10-15T04:56:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:04.524+00:00 2026-10-15T04:56:04Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:04.524+00:00 2026-10-15T04:56:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:05.528+00:00 2026-10-15T04:56:05Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:05.528+00:00 2026-10-15T04:56:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:06.549+00:00 2026-10-15T04:56:06Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:06.549+00:00 2026-10-15T04:56:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:07.539+00:00 2026-10-15T04:56:07Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:07.539+00:00 2026-10-15T04:56:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:08.558+00:00 2026-10-15T04:56:08Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:08.558+00:00 2026-10-15T04:56:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:09.549+00:00 2026-10-15T04:56:09Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:09.550+00:00 2026-10-15T04:56:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:10.555+00:00 2026-10-15T04:56:10Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:56:10.555+00:00 2026-10-15T04:56:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log deleted file mode 100644 index 78eb0719..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T04:47:52.102+00:00 2026-10-15T04:47:52.104023958Z {"level":"info","ts":"2026-10-15T04:47:52Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"8483800e-5d84-4a39-9698-b4f98903534e","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} -2026-10-15T04:51:15.356+00:00 2026-10-15T04:51:15.366719534Z {"level":"info","ts":"2026-10-15T04:51:15Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"a33a6ae4-616b-4e2d-b00c-0100362558de","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json deleted file mode 100644 index fbc28464..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/timeline.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "case": "after-A1-0.5b-l4-cold", - "collected": "2026-10-15T04:56:11.555+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 04:47:51.110000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen2.5-0.5B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 04:47:51.111000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 04:51:08.186000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "c34fd223-1dd8-443e-850e-68d16b0d897e" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 04:51:10.937000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 04:51:14.170000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 04:51:15.072000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 04:51:15.072000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 04:51:15.661000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 04:51:15.662000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 04:56:04.835000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4" - }, - "client.end": { - "ts": "2026-10-15 04:56:04.838000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 04:51:15+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 04:51:15.069000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 04:51:15.069000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 04:51:15.069000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 04:54:22+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 04:51:15+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 04:51:15+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 04:54:22+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 04:54:22+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 04:51:15+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 04:54:22+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 04:51:15+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "sampler.event_Pulling": { - "ts": "2026-10-15 04:51:19+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 04:54:21+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s inc", - "kubelet_pull_s": 182.5, - "pulled_from_cache": false - }, - "sampler.event_Created": { - "ts": "2026-10-15 04:54:21+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 04:54:22+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 04:54:45.242588+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 04:55:00.403987+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:00 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 04:55:02.093177+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:02 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 04:55:03.439031+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:03 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 04:55:04.519608+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:04 [default_loader.py:430] Loading weights took 1.08 seconds", - "self_reported_s": 1.08 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 04:55:05.323053+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:05 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.543823 seconds", - "self_reported_s": 3.543823 - }, - "sampler.compile_done": { - "ts": "2026-10-15 04:55:36.056714+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:36 [monitor.py:53] torch.compile took 28.53 s in total", - "self_reported_s": 28.53 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 04:55:51.460006+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:51 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", - "self_reported_s": 11.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 04:55:52.012063+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:55:52 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 04:56:02.777706+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:56:02 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.45 s", - "self_reported_s": 57.45 - }, - "sampler.ready_key": { - "ts": "2026-10-15 04:56:04+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - }, - "trainer": { - "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 04:47:51+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 04:47:51.664000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 04:47:51.664000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 04:47:53.304000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 04:50:45+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 04:47:52+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 04:47:52+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 04:50:45+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 04:50:45+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 04:47:52+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 04:50:45+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 04:47:52+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "trainer.event_Pulling": { - "ts": "2026-10-15 04:47:56+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 04:50:44+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.27", - "kubelet_pull_s": 168.27100000000002, - "pulled_from_cache": false - }, - "trainer.event_Created": { - "ts": "2026-10-15 04:50:44+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 04:50:45+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 04:50:52.077847+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 04:50:52.221389+00:00", - "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 04:51:07.261679+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 04:51:07.264149+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 04:51:08+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt deleted file mode 100644 index e34900e5..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T04:47:46.873+00:00 diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt deleted file mode 100644 index a9917932..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json b/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json deleted file mode 100644 index c15af16a..00000000 --- a/runs/2026-10-15_cold_start/after-A1-0.5b-l4-cold/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040062551935001", - "uid": "0752a694-3e75-44aa-9348-ae1c3f3fff13" - }, - "spec": { - "accelerator": { - "memory": "4Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "ed5c4cfd-e94a-4a7f-a2a6-62ba293d8d04", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T04:54:22Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-0752a694", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "4Gi", - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:51Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039845830863015", - "uid": "4bef5a35-3bf8-4444-9043-47f86b502e3e" - }, - "spec": { - "accelerator": { - "memory": "5Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "0772feaa-10d8-4243-ac6a-4b4ee0615042", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T04:50:45Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-4bef5a35", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "5Gi", - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log deleted file mode 100644 index 8bf3bc56..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached.run.log +++ /dev/null @@ -1,35 +0,0 @@ -watching; recorders [14913, 14914, 14916, 14918, 14920, 14915, 14917, 14919] write into runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached -collected 2 worker log(s) into runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.2 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 1.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 3.0 | -| container create | trainer.event_Pulled | trainer.container_started | 0.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 7.8 | -| weights load | trainer.load_start | trainer.load_done | 2.5 | -| ready key | trainer.load_done | trainer.ready_key | -18.3 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 15.5 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.0 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 29.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 3.0 | -| container create | sampler.event_Pulled | sampler.container_started | 0.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 11.6 | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 27.1 | -| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.7 | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.5 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.4 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.9 | -| ready key | sampler.engine_init_done | sampler.ready_key | 0.9 | -| client: first sample blocked | client.sample.start | client.sample.done | 134.6 | - diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log deleted file mode 100644 index 837468ea..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/api-server.log +++ /dev/null @@ -1,33 +0,0 @@ -2026-10-15T04:57:20.742+00:00 2026-10-15T04:57:20.747394853Z INFO: 10.32.3.1:48902 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:57:22.081+00:00 2026-10-15T04:57:22.098120271Z INFO: 127.0.0.1:51376 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T04:57:22.332+00:00 2026-10-15T04:57:22.350492601Z INFO: 127.0.0.1:51380 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T04:57:22.399+00:00 2026-10-15T04:57:22.417496524Z INFO: 127.0.0.1:51380 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T04:57:22.642+00:00 2026-10-15T04:57:22.659983411Z [API_SERVER] enqueued op=create_model request_id=3c646264-eb51-4f2b-af28-b651e7b8d65f model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:57:22.642+00:00 2026-10-15T04:57:22.660591970Z INFO: 127.0.0.1:51390 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T04:57:30.732+00:00 2026-10-15T04:57:30.746832525Z INFO: 10.32.3.1:60502 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:57:32.582+00:00 2026-10-15T04:57:32.599215799Z INFO: 127.0.0.1:41916 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:57:37.537+00:00 2026-10-15T04:57:37.550418481Z INFO: 127.0.0.1:41936 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T04:57:40.140+00:00 2026-10-15T04:57:40.156908336Z [API_SERVER] enqueued op=forward_backward request_id=e5d06b8f-011b-4cc7-b00f-75a18d4cc68c model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:57:40.140+00:00 2026-10-15T04:57:40.157358753Z INFO: 127.0.0.1:41936 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T04:57:40.728+00:00 2026-10-15T04:57:40.746424135Z INFO: 10.32.3.1:52382 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:57:43.347+00:00 2026-10-15T04:57:43.364027632Z [API_SERVER] enqueued op=optim_step request_id=9c7bfa9d-8c2e-4163-8ddb-2d3872d13f25 model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:57:43.347+00:00 2026-10-15T04:57:43.364731477Z INFO: 127.0.0.1:41936 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T04:57:44.137+00:00 2026-10-15T04:57:44.154771060Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=32c6fd95-8437-4062-86d2-5fd031af54e4 model_id=3c646264-eb51-4f2b-af28-b651e7b8d65f active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T04:57:44.138+00:00 2026-10-15T04:57:44.156034597Z INFO: 127.0.0.1:41936 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T04:57:44.807+00:00 2026-10-15T04:57:44.824246114Z INFO: 127.0.0.1:35608 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T04:57:50.742+00:00 2026-10-15T04:57:50.746409736Z INFO: 10.32.3.1:46740 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:58:00.732+00:00 2026-10-15T04:58:00.747039608Z INFO: 10.32.3.1:49148 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:58:10.732+00:00 2026-10-15T04:58:10.747575732Z INFO: 10.32.3.1:53698 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:58:20.731+00:00 2026-10-15T04:58:20.747432364Z INFO: 10.32.3.1:58312 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:58:30.731+00:00 2026-10-15T04:58:30.747504088Z INFO: 10.32.3.1:52920 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:58:32.787+00:00 2026-10-15T04:58:32.804434225Z INFO: 127.0.0.1:46778 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:58:40.730+00:00 2026-10-15T04:58:40.746974548Z INFO: 10.32.3.1:46976 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:58:50.730+00:00 2026-10-15T04:58:50.747141544Z INFO: 10.32.3.1:59494 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:59:00.731+00:00 2026-10-15T04:59:00.747576147Z INFO: 10.32.3.1:43150 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:59:10.730+00:00 2026-10-15T04:59:10.747345321Z INFO: 10.32.3.1:32882 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:59:20.732+00:00 2026-10-15T04:59:20.747511163Z INFO: 10.32.3.1:54978 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:59:22.982+00:00 2026-10-15T04:59:23.000147720Z INFO: 127.0.0.1:39214 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T04:59:30.737+00:00 2026-10-15T04:59:30.746624775Z INFO: 10.32.3.1:33562 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:59:40.731+00:00 2026-10-15T04:59:40.747472732Z INFO: 10.32.3.1:56128 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T04:59:50.729+00:00 2026-10-15T04:59:50.746593670Z INFO: 10.32.3.1:54218 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:00:00.731+00:00 2026-10-15T05:00:00.747026021Z INFO: 10.32.3.1:37274 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl deleted file mode 100644 index 6f50fdd1..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T04:57:21.804+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T04:57:21.804+00:00", "t": 0.001, "phase": "create_model.start"} -{"ts": "2026-10-15T04:57:37.299+00:00", "t": 15.495, "phase": "create_model.done", "model_id": "3c646264-eb51-4f2b-af28-b651e7b8d65f"} -{"ts": "2026-10-15T04:57:40.066+00:00", "t": 18.262, "phase": "forward_backward.start"} -{"ts": "2026-10-15T04:57:43.279+00:00", "t": 21.475, "phase": "forward_backward.done"} -{"ts": "2026-10-15T04:57:44.032+00:00", "t": 22.228, "phase": "optim_step.done"} -{"ts": "2026-10-15T04:57:44.033+00:00", "t": 22.229, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T04:57:44.610+00:00", "t": 22.807, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T04:57:44.612+00:00", "t": 22.808, "phase": "sample.start"} -{"ts": "2026-10-15T04:59:59.246+00:00", "t": 157.443, "phase": "sample.done", "text": " 4"} -{"ts": "2026-10-15T04:59:59.247+00:00", "t": 157.443, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log deleted file mode 100644 index 8738ab7c..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.001s create_model.start -[cold-start] 15.495s create_model.done {'model_id': '3c646264-eb51-4f2b-af28-b651e7b8d65f'} -[cold-start] 18.262s forward_backward.start -[cold-start] 21.475s forward_backward.done -[cold-start] 22.228s optim_step.done -[cold-start] 22.229s save_weights_and_get_sampling_client.start -[cold-start] 22.807s save_weights_and_get_sampling_client.done -[cold-start] 22.808s sample.start -[cold-start] 157.443s sample.done {'text': ' 4'} -[cold-start] 157.443s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt deleted file mode 100644 index 2cb3aa22..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T04:59:59.733+00:00 diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json deleted file mode 100644 index f67d42a0..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/events.json +++ /dev/null @@ -1,1699 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:10Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:10Z", - "name": "cold-start-fio.18de9a0ef295773d", - "namespace": "openrl-system", - "resourceVersion": "1792039090760383011", - "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5be63862", - "namespace": "openrl-system", - "resourceVersion": "1792039092515007000", - "uid": "355b59b9-3661-499d-8640-3f40c3c89375" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5ecfa51a", - "namespace": "openrl-system", - "resourceVersion": "1792039092561663021", - "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f61ddd1e4", - "namespace": "openrl-system", - "resourceVersion": "1792039092605935011", - "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:40:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:40:41Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:40:41Z", - "name": "cold-start-fio.18de9a32081cf908", - "namespace": "openrl-system", - "resourceVersion": "1792039241446287009", - "uid": "14a62e42-13ba-4526-a04d-a934702e1163" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:18Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:18Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:18Z", - "name": "cold-start-fio.18de9a3a9867ce9f", - "namespace": "openrl-system", - "resourceVersion": "1792039278208207001", - "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad2bcac91", - "namespace": "openrl-system", - "resourceVersion": "1792039279188751014", - "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad378cd74", - "namespace": "openrl-system", - "resourceVersion": "1792039279219679006", - "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad5aed221", - "namespace": "openrl-system", - "resourceVersion": "1792039279235135013", - "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:11Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:11Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:11Z", - "name": "cold-start-fio.18de9a70cd2c6a68", - "namespace": "openrl-system", - "resourceVersion": "1792039511048479001", - "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "resourceVersion": "1791941114537823021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Stopping container api-server", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", - "namespace": "openrl-system", - "resourceVersion": "1792039545418735013", - "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "namespace": "openrl-system", - "resourceVersion": "1792039545300351007", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", - "namespace": "openrl-system", - "resourceVersion": "1792039545387567001", - "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" - }, - "reason": "SuccessfulDelete", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546048591003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", - "namespace": "openrl-system", - "resourceVersion": "1792039546102351012", - "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:49Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:49Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", - "namespace": "openrl-system", - "resourceVersion": "1792039549712335019", - "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", - "namespace": "openrl-system", - "resourceVersion": "1792039558721215017", - "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", - "namespace": "openrl-system", - "resourceVersion": "1792039558783199013", - "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", - "namespace": "openrl-system", - "resourceVersion": "1792039558862303009", - "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "namespace": "openrl-system", - "resourceVersion": "1792039545940671018", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", - "namespace": "openrl-system", - "resourceVersion": "1792039546068975009", - "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" - }, - "reason": "SuccessfulCreate", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545232655018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78c9668cf7", - "namespace": "openrl-system", - "resourceVersion": "1792039545327119005", - "uid": "7b1433df-3524-478b-aba2-8979950e245e" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545464319018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78ef8b9e1b", - "namespace": "openrl-system", - "resourceVersion": "1792039545956079020", - "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:15Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875490623001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:15Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", - "namespace": "openrl-system", - "resourceVersion": "1792039875609279013", - "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:19Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:19Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", - "namespace": "openrl-system", - "resourceVersion": "1792039879126319008", - "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", - "namespace": "openrl-system", - "resourceVersion": "1792040061645247020", - "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", - "namespace": "openrl-system", - "resourceVersion": "1792040061849391002", - "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", - "namespace": "openrl-system", - "resourceVersion": "1792040062252079023", - "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", - "namespace": "openrl-system", - "resourceVersion": "1792040192780783021", - "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264403887001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", - "namespace": "openrl-system", - "resourceVersion": "1792040264444463021", - "uid": "a85006a2-68a6-4919-b148-08d58b546d07" - }, - "reason": "FailedScheduling", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Warning" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264434463001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", - "namespace": "openrl-system", - "resourceVersion": "1792040264884223002", - "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" - }, - "reason": "NotTriggerScaleUp", - "reportingComponent": "cluster-autoscaler", - "reportingInstance": "", - "source": { - "component": "cluster-autoscaler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264893039001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", - "namespace": "openrl-system", - "resourceVersion": "1792040293171567001", - "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", - "namespace": "openrl-system", - "resourceVersion": "1792040296672127014", - "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", - "namespace": "openrl-system", - "resourceVersion": "1792040296703007017", - "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", - "namespace": "openrl-system", - "resourceVersion": "1792040296889487014", - "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:52Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672220015012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:52Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", - "namespace": "openrl-system", - "resourceVersion": "1792039672350591019", - "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:56Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:56Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:56Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", - "namespace": "openrl-system", - "resourceVersion": "1792039676166239015", - "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", - "namespace": "openrl-system", - "resourceVersion": "1792039844457167000", - "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", - "namespace": "openrl-system", - "resourceVersion": "1792039844632591021", - "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:45Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:45Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", - "namespace": "openrl-system", - "resourceVersion": "1792039845170959016", - "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", - "namespace": "openrl-system", - "resourceVersion": "1792040192981183013", - "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:23Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040242926223012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:23Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:23Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", - "namespace": "openrl-system", - "resourceVersion": "1792040243043743024", - "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", - "namespace": "openrl-system", - "resourceVersion": "1792040246545487023", - "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", - "namespace": "openrl-system", - "resourceVersion": "1792040246576895006", - "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", - "namespace": "openrl-system", - "resourceVersion": "1792040246742815002", - "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", - "namespace": "openrl-system", - "resourceVersion": "1792038835732623016", - "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", - "namespace": "openrl-system", - "resourceVersion": "1792038835415695012", - "uid": "918abc9a-1f24-4767-8928-b202482f11ed" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt deleted file mode 100644 index 0b795ec8..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt +++ /dev/null @@ -1,119 +0,0 @@ -2026-10-15T04:58:27.584176783Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T04:58:39.659845746Z INFO 10-15 04:58:39 [model.py:692] Resolved architecture: Qwen2ForCausalLM -2026-10-15T04:58:39.660367167Z INFO 10-15 04:58:39 [model.py:2030] Using max model len 8192 -2026-10-15T04:58:39.662546840Z INFO 10-15 04:58:39 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. -2026-10-15T04:58:40.049753461Z INFO 10-15 04:58:40 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T04:58:43.331466819Z WARNING 10-15 04:58:43 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T04:58:54.733167125Z (EngineCore pid=161) INFO 10-15 04:58:54 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T04:58:55.532460244Z (EngineCore pid=161) INFO 10-15 04:58:55 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_8fc70d1b7a3d4bf98e7322fa6a9dbb46 backend=nccl -2026-10-15T04:58:55.539943546Z (EngineCore pid=161) INFO 10-15 04:58:55 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T04:58:55.539970703Z (EngineCore pid=161) INFO 10-15 04:58:55 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T04:58:56.402021183Z (EngineCore pid=161) INFO 10-15 04:58:56 [model_runner.py:396] Loading model from scratch... -2026-10-15T04:58:57.269734349Z (EngineCore pid=161) INFO 10-15 04:58:57 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T04:58:57.270211979Z (EngineCore pid=161) INFO 10-15 04:58:57 [flash_attn.py:1116] Using FlashAttention version 2 -2026-10-15T04:58:57.386036165Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T04:58:57.386373759Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T04:58:57.753318801Z (EngineCore pid=161) INFO 10-15 04:58:57 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.62 GiB. -2026-10-15T04:58:57.753358302Z (EngineCore pid=161) INFO 10-15 04:58:57 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T04:58:57.754786362Z (EngineCore pid=161) -Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T04:57:33.682578439Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T04:57:33.682611900Z -2026-10-15T04:57:33.802165728Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... -2026-10-15T04:57:33.918116197Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T04:57:34.743979630Z [prefetch] reading 1 file(s), 0.92 GiB, from nfs into the page cache with 8 threads -2026-10-15T04:57:35.446752761Z [prefetch] done: 0.92 GiB in 0.7 s (1341 MiB/s) -2026-10-15T04:57:36.264734680Z -Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json deleted file mode 100644 index 6a4ea21e..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/pods.json +++ /dev/null @@ -1,6786 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "generateName": "open-rl-api-server-6d86b498f5-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "6d86b498f5", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - } - ], - "resourceVersion": "1792039570790271003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2ns79", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:59Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:45:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.11", - "podIPs": [ - { - "ip": "10.32.3.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:45:46Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "cloud.google.com/cluster_autoscaler_unhelpable_since": "2026-10-15T04:57:44+0000", - "cloud.google.com/cluster_autoscaler_unhelpable_until": "Inf" - }, - "creationTimestamp": "2026-10-15T04:57:44Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "uid": "4ce140e7-4f1e-4b14-9877-bb659f6e0343" - } - ], - "resourceVersion": "1792040297125103001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "2ee5c8bd-e346-48b6-8cb1-eb62a873538d" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j4zlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-j4zlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:58:17Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:58:13Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:58:17Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:58:17Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:58:13Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "25Gi" - }, - "containerID": "containerd://9da0479c74bfc8042a71c292922826c6ce3ad90ac3ac5cb3b018c1f4cff943a0", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:58:16Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j4zlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.0.21", - "podIPs": [ - { - "ip": "10.32.0.21" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:58:13Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:22Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "uid": "35aa092e-27db-4eca-ad32-e9e0d6070c77" - } - ], - "resourceVersion": "1792040246938655012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "efac33f1-a138-4d61-b066-ff1ef1279fec" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-gvbfv", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-gvbfv", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:57:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:57:23Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:57:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:57:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:57:23Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "21Gi" - }, - "containerID": "containerd://2d4343c24b9a9775657cfe3e8106b8ddd769515df55d3f40bbcc45bca83bbd69", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:57:26Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-gvbfv", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.13.27", - "podIPs": [ - { - "ip": "10.32.13.27" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:57:23Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log deleted file mode 100644 index d60c3560..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/redis-ready.log +++ /dev/null @@ -1,175 +0,0 @@ -2026-10-15T04:57:18.742+00:00 2026-10-15T04:57:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:19.743+00:00 2026-10-15T04:57:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:20.748+00:00 2026-10-15T04:57:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:21.753+00:00 2026-10-15T04:57:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:22.758+00:00 2026-10-15T04:57:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:23.763+00:00 2026-10-15T04:57:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:24.768+00:00 2026-10-15T04:57:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:25.774+00:00 2026-10-15T04:57:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:26.778+00:00 2026-10-15T04:57:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:27.784+00:00 2026-10-15T04:57:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:28.790+00:00 2026-10-15T04:57:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:29.796+00:00 2026-10-15T04:57:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:30.800+00:00 2026-10-15T04:57:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:31.806+00:00 2026-10-15T04:57:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:32.811+00:00 2026-10-15T04:57:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:33.815+00:00 2026-10-15T04:57:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:34.820+00:00 2026-10-15T04:57:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:35.826+00:00 2026-10-15T04:57:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:36.830+00:00 2026-10-15T04:57:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:37.835+00:00 2026-10-15T04:57:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:38.840+00:00 2026-10-15T04:57:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:39.846+00:00 2026-10-15T04:57:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:40.851+00:00 2026-10-15T04:57:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:41.857+00:00 2026-10-15T04:57:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:42.862+00:00 2026-10-15T04:57:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:43.866+00:00 2026-10-15T04:57:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:44.870+00:00 2026-10-15T04:57:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:45.875+00:00 2026-10-15T04:57:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:46.879+00:00 2026-10-15T04:57:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:47.906+00:00 2026-10-15T04:57:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:48.893+00:00 2026-10-15T04:57:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:49.897+00:00 2026-10-15T04:57:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:50.901+00:00 2026-10-15T04:57:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:51.907+00:00 2026-10-15T04:57:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:52.911+00:00 2026-10-15T04:57:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:53.917+00:00 2026-10-15T04:57:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:54.921+00:00 2026-10-15T04:57:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:55.928+00:00 2026-10-15T04:57:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:56.946+00:00 2026-10-15T04:57:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:57.941+00:00 2026-10-15T04:57:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:58.944+00:00 2026-10-15T04:57:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:57:59.953+00:00 2026-10-15T04:57:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:00.953+00:00 2026-10-15T04:58:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:01.958+00:00 2026-10-15T04:58:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:02.972+00:00 2026-10-15T04:58:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:03.974+00:00 2026-10-15T04:58:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:04.981+00:00 2026-10-15T04:58:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:05.980+00:00 2026-10-15T04:58:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:06.985+00:00 2026-10-15T04:58:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:07.991+00:00 2026-10-15T04:58:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:08.996+00:00 2026-10-15T04:58:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:10.027+00:00 2026-10-15T04:58:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:11.015+00:00 2026-10-15T04:58:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:12.011+00:00 2026-10-15T04:58:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:13.099+00:00 2026-10-15T04:58:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:14.021+00:00 2026-10-15T04:58:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:15.026+00:00 2026-10-15T04:58:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:16.031+00:00 2026-10-15T04:58:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:17.206+00:00 2026-10-15T04:58:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:18.041+00:00 2026-10-15T04:58:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:19.059+00:00 2026-10-15T04:58:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:20.052+00:00 2026-10-15T04:58:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:21.062+00:00 2026-10-15T04:58:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:22.064+00:00 2026-10-15T04:58:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:23.067+00:00 2026-10-15T04:58:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:24.077+00:00 2026-10-15T04:58:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:25.078+00:00 2026-10-15T04:58:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:26.110+00:00 2026-10-15T04:58:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:27.093+00:00 2026-10-15T04:58:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:28.097+00:00 2026-10-15T04:58:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:29.108+00:00 2026-10-15T04:58:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:30.111+00:00 2026-10-15T04:58:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:31.112+00:00 2026-10-15T04:58:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:32.117+00:00 2026-10-15T04:58:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:33.122+00:00 2026-10-15T04:58:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:34.127+00:00 2026-10-15T04:58:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:35.171+00:00 2026-10-15T04:58:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:36.137+00:00 2026-10-15T04:58:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:37.143+00:00 2026-10-15T04:58:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:38.148+00:00 2026-10-15T04:58:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:39.153+00:00 2026-10-15T04:58:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:40.161+00:00 2026-10-15T04:58:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:41.163+00:00 2026-10-15T04:58:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:42.168+00:00 2026-10-15T04:58:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:43.173+00:00 2026-10-15T04:58:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:44.179+00:00 2026-10-15T04:58:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:45.183+00:00 2026-10-15T04:58:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:46.290+00:00 2026-10-15T04:58:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:47.193+00:00 2026-10-15T04:58:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:48.198+00:00 2026-10-15T04:58:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:49.204+00:00 2026-10-15T04:58:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:50.208+00:00 2026-10-15T04:58:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:51.394+00:00 2026-10-15T04:58:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:52.218+00:00 2026-10-15T04:58:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:53.225+00:00 2026-10-15T04:58:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:54.241+00:00 2026-10-15T04:58:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:55.234+00:00 2026-10-15T04:58:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:56.239+00:00 2026-10-15T04:58:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:57.245+00:00 2026-10-15T04:58:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:58.257+00:00 2026-10-15T04:58:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:58:59.256+00:00 2026-10-15T04:58:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:00.269+00:00 2026-10-15T04:59:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:01.271+00:00 2026-10-15T04:59:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:02.271+00:00 2026-10-15T04:59:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:03.287+00:00 2026-10-15T04:59:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:04.283+00:00 2026-10-15T04:59:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:05.286+00:00 2026-10-15T04:59:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:06.291+00:00 2026-10-15T04:59:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:07.296+00:00 2026-10-15T04:59:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:08.300+00:00 2026-10-15T04:59:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:09.310+00:00 2026-10-15T04:59:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:10.311+00:00 2026-10-15T04:59:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:11.316+00:00 2026-10-15T04:59:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:12.322+00:00 2026-10-15T04:59:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:13.327+00:00 2026-10-15T04:59:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:14.331+00:00 2026-10-15T04:59:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:15.337+00:00 2026-10-15T04:59:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:16.344+00:00 2026-10-15T04:59:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:17.348+00:00 2026-10-15T04:59:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:18.352+00:00 2026-10-15T04:59:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:19.357+00:00 2026-10-15T04:59:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:20.366+00:00 2026-10-15T04:59:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:21.368+00:00 2026-10-15T04:59:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:22.377+00:00 2026-10-15T04:59:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:23.379+00:00 2026-10-15T04:59:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:24.384+00:00 2026-10-15T04:59:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:25.390+00:00 2026-10-15T04:59:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:26.395+00:00 2026-10-15T04:59:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:27.403+00:00 2026-10-15T04:59:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:28.404+00:00 2026-10-15T04:59:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:29.410+00:00 2026-10-15T04:59:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:30.415+00:00 2026-10-15T04:59:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:31.420+00:00 2026-10-15T04:59:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:32.426+00:00 2026-10-15T04:59:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:33.432+00:00 2026-10-15T04:59:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:34.437+00:00 2026-10-15T04:59:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:35.445+00:00 2026-10-15T04:59:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:36.449+00:00 2026-10-15T04:59:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:37.453+00:00 2026-10-15T04:59:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:38.458+00:00 2026-10-15T04:59:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:39.463+00:00 2026-10-15T04:59:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:40.469+00:00 2026-10-15T04:59:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:41.474+00:00 2026-10-15T04:59:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:42.478+00:00 2026-10-15T04:59:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:43.484+00:00 2026-10-15T04:59:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:44.490+00:00 2026-10-15T04:59:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:45.494+00:00 2026-10-15T04:59:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:46.500+00:00 2026-10-15T04:59:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:47.508+00:00 2026-10-15T04:59:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:48.509+00:00 2026-10-15T04:59:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:49.514+00:00 2026-10-15T04:59:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:50.519+00:00 2026-10-15T04:59:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:51.532+00:00 2026-10-15T04:59:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:52.529+00:00 2026-10-15T04:59:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:53.535+00:00 2026-10-15T04:59:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:54.545+00:00 2026-10-15T04:59:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:55.544+00:00 2026-10-15T04:59:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:56.565+00:00 2026-10-15T04:59:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:57.556+00:00 2026-10-15T04:59:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:58.561+00:00 2026-10-15T04:59:58Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:58.562+00:00 2026-10-15T04:59:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:59.566+00:00 2026-10-15T04:59:59Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T04:59:59.566+00:00 2026-10-15T04:59:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:00.571+00:00 2026-10-15T05:00:00Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:00.571+00:00 2026-10-15T05:00:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:01.590+00:00 2026-10-15T05:00:01Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:01.590+00:00 2026-10-15T05:00:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:02.582+00:00 2026-10-15T05:00:02Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:02.582+00:00 2026-10-15T05:00:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:03.588+00:00 2026-10-15T05:00:03Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:03.588+00:00 2026-10-15T05:00:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:04.600+00:00 2026-10-15T05:00:04Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:04.600+00:00 2026-10-15T05:00:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:05.599+00:00 2026-10-15T05:00:05Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:00:05.599+00:00 2026-10-15T05:00:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log deleted file mode 100644 index 0a6b928c..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T04:57:22.772+00:00 2026-10-15T04:57:22.789786341Z {"level":"info","ts":"2026-10-15T04:57:22Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"a299badc-c2da-43ab-b006-1bbc6eef5dd0","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} -2026-10-15T04:57:44.281+00:00 2026-10-15T04:57:44.297614785Z {"level":"info","ts":"2026-10-15T04:57:44Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"ac3b61c9-8905-4726-9ca8-f88e8b72698f","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json deleted file mode 100644 index d90e4c40..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/timeline.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "case": "after-B1-0.5b-l4-cached", - "collected": "2026-10-15T05:00:05.835+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 04:57:21.804000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen2.5-0.5B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 04:57:21.804000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 04:57:37.299000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "3c646264-eb51-4f2b-af28-b651e7b8d65f" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 04:57:40.066000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 04:57:43.279000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 04:57:44.032000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 04:57:44.033000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 04:57:44.610000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 04:57:44.612000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 04:59:59.246000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4" - }, - "client.end": { - "ts": "2026-10-15 04:59:59.247000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 04:57:44+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 04:57:44.625000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 04:57:44.625000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 04:58:13.579000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 04:58:17+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 04:57:44+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 04:58:13+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 04:58:17+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 04:58:17+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 04:58:13+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 04:58:16+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_FailedScheduling": { - "ts": "2026-10-15 04:57:44+00:00", - "source": "event: 0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, p" - }, - "sampler.event_NotTriggerScaleUp": { - "ts": "2026-10-15 04:57:44+00:00", - "source": "event: Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 04:58:13+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 04:58:16+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "sampler.event_Created": { - "ts": "2026-10-15 04:58:16+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 04:58:16+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 04:58:27.584176+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 04:58:54.733167+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:54 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 04:58:56.402021+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:56 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 04:58:57.753358+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:57 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 04:58:58.859754+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:58 [default_loader.py:430] Loading weights took 1.11 seconds", - "self_reported_s": 1.11 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 04:58:59.653997+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:58:59 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.562917 seconds", - "self_reported_s": 3.562917 - }, - "sampler.compile_done": { - "ts": "2026-10-15 04:59:30.263531+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:30 [monitor.py:53] torch.compile took 28.39 s in total", - "self_reported_s": 28.39 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 04:59:45.735209+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:45 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", - "self_reported_s": 11.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 04:59:46.273450+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:46 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 04:59:57.142509+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 04:59:57 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.48 s", - "self_reported_s": 57.48 - }, - "sampler.ready_key": { - "ts": "2026-10-15 04:59:58+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - }, - "trainer": { - "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 04:57:22+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 04:57:22.498000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 04:57:22.498000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 04:57:22.498000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 04:57:27+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 04:57:22+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 04:57:23+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 04:57:26+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 04:57:26+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 04:57:23+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 04:57:26+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 04:57:23+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 04:57:26+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "trainer.event_Created": { - "ts": "2026-10-15 04:57:26+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 04:57:26+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 04:57:33.659925+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 04:57:33.802165+00:00", - "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 04:57:36.346688+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 04:57:36.349344+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 04:57:18+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt deleted file mode 100644 index 6982d578..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T04:57:17.557+00:00 diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt deleted file mode 100644 index a9917932..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json b/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json deleted file mode 100644 index 8e52f6bf..00000000 --- a/runs/2026-10-15_cold_start/after-B1-0.5b-l4-cached/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040297218207001", - "uid": "4ce140e7-4f1e-4b14-9877-bb659f6e0343" - }, - "spec": { - "accelerator": { - "memory": "4Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "2ee5c8bd-e346-48b6-8cb1-eb62a873538d", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T04:58:17Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-4ce140e7", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "4Gi", - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:22Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040247036111015", - "uid": "35aa092e-27db-4eca-ad32-e9e0d6070c77" - }, - "spec": { - "accelerator": { - "memory": "5Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "efac33f1-a138-4d61-b066-ff1ef1279fec", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T04:57:27Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-35aa092e", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "5Gi", - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log deleted file mode 100644 index 7473ff1d..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached.run.log +++ /dev/null @@ -1,35 +0,0 @@ -watching; recorders [15841, 15842, 15844, 15846, 15848, 15843, 15845, 15847] write into runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached -collected 2 worker log(s) into runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.4 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | -| container create | trainer.event_Pulled | trainer.container_started | 0.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 8.0 | -| weights load | trainer.load_start | trainer.load_done | 2.5 | -| ready key | trainer.load_done | trainer.ready_key | -18.5 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 15.8 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 3.2 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.2 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | -| container create | sampler.event_Pulled | sampler.container_started | 0.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 23.3 | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 16.3 | -| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.8 | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 2.6 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 31.3 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 26.8 | -| ready key | sampler.engine_init_done | sampler.ready_key | 1.9 | -| client: first sample blocked | client.sample.start | client.sample.done | 107.4 | - diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log deleted file mode 100644 index df117526..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/api-server.log +++ /dev/null @@ -1,30 +0,0 @@ -2026-10-15T05:01:15.885+00:00 2026-10-15T05:01:15.899417391Z INFO: 127.0.0.1:35950 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T05:01:16.112+00:00 2026-10-15T05:01:16.130561464Z INFO: 127.0.0.1:35952 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T05:01:16.180+00:00 2026-10-15T05:01:16.199505092Z INFO: 127.0.0.1:35952 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T05:01:16.443+00:00 2026-10-15T05:01:16.462704698Z [API_SERVER] enqueued op=create_model request_id=840431b4-6617-4074-8848-cf95526c9aec model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T05:01:16.445+00:00 2026-10-15T05:01:16.464180264Z INFO: 127.0.0.1:35962 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T05:01:20.728+00:00 2026-10-15T05:01:20.746712614Z INFO: 10.32.3.1:43472 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:01:26.372+00:00 2026-10-15T05:01:26.389349362Z INFO: 127.0.0.1:48808 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:01:30.732+00:00 2026-10-15T05:01:30.748033533Z INFO: 10.32.3.1:45936 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:01:31.605+00:00 2026-10-15T05:01:31.624783860Z INFO: 127.0.0.1:42044 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T05:01:34.338+00:00 2026-10-15T05:01:34.357162387Z [API_SERVER] enqueued op=forward_backward request_id=d985e84c-f973-4fc5-84a8-b078e63ccf69 model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T05:01:34.339+00:00 2026-10-15T05:01:34.357733525Z INFO: 127.0.0.1:42044 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T05:01:37.559+00:00 2026-10-15T05:01:37.576296735Z [API_SERVER] enqueued op=optim_step request_id=d3a96a54-0418-44fe-a5c3-2f1e6be71f7b model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T05:01:37.559+00:00 2026-10-15T05:01:37.576838072Z INFO: 127.0.0.1:42044 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T05:01:38.342+00:00 2026-10-15T05:01:38.358865914Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=963d0bbf-ae16-4566-b427-cfcb9f37e811 model_id=840431b4-6617-4074-8848-cf95526c9aec active_set=Qwen/Qwen2.5-0.5B-1 -2026-10-15T05:01:38.342+00:00 2026-10-15T05:01:38.359362687Z INFO: 127.0.0.1:42044 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T05:01:39.001+00:00 2026-10-15T05:01:39.020317120Z INFO: 127.0.0.1:42052 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T05:01:40.729+00:00 2026-10-15T05:01:40.747481210Z INFO: 10.32.3.1:52820 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:01:50.730+00:00 2026-10-15T05:01:50.747013214Z INFO: 10.32.3.1:36308 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:02:00.734+00:00 2026-10-15T05:02:00.746899682Z INFO: 10.32.3.1:34568 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:02:10.729+00:00 2026-10-15T05:02:10.747400430Z INFO: 10.32.3.1:38018 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:02:20.729+00:00 2026-10-15T05:02:20.747204752Z INFO: 10.32.3.1:54932 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:02:26.583+00:00 2026-10-15T05:02:26.601098990Z INFO: 127.0.0.1:41122 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:02:30.730+00:00 2026-10-15T05:02:30.748457327Z INFO: 10.32.3.1:33506 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:02:40.729+00:00 2026-10-15T05:02:40.747125356Z INFO: 10.32.3.1:49354 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:02:50.733+00:00 2026-10-15T05:02:50.747387509Z INFO: 10.32.3.1:41164 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:03:00.728+00:00 2026-10-15T05:03:00.746612139Z INFO: 10.32.3.1:37988 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:03:10.728+00:00 2026-10-15T05:03:10.746728182Z INFO: 10.32.3.1:35818 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:03:16.776+00:00 2026-10-15T05:03:16.783574019Z INFO: 127.0.0.1:58382 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:03:20.727+00:00 2026-10-15T05:03:20.746403654Z INFO: 10.32.3.1:41964 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:03:30.821+00:00 2026-10-15T05:03:30.746892009Z INFO: 10.32.3.1:57042 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl deleted file mode 100644 index 15a11f70..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T05:01:15.634+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen2.5-0.5B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T05:01:15.634+00:00", "t": 0.001, "phase": "create_model.start"} -{"ts": "2026-10-15T05:01:31.419+00:00", "t": 15.785, "phase": "create_model.done", "model_id": "840431b4-6617-4074-8848-cf95526c9aec"} -{"ts": "2026-10-15T05:01:34.271+00:00", "t": 18.637, "phase": "forward_backward.start"} -{"ts": "2026-10-15T05:01:37.484+00:00", "t": 21.85, "phase": "forward_backward.done"} -{"ts": "2026-10-15T05:01:38.241+00:00", "t": 22.607, "phase": "optim_step.done"} -{"ts": "2026-10-15T05:01:38.245+00:00", "t": 22.611, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T05:01:38.818+00:00", "t": 23.184, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T05:01:38.819+00:00", "t": 23.185, "phase": "sample.start"} -{"ts": "2026-10-15T05:03:26.180+00:00", "t": 130.546, "phase": "sample.done", "text": " 4"} -{"ts": "2026-10-15T05:03:26.181+00:00", "t": 130.547, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log deleted file mode 100644 index ee6051b8..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen2.5-0.5B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.001s create_model.start -[cold-start] 15.785s create_model.done {'model_id': '840431b4-6617-4074-8848-cf95526c9aec'} -[cold-start] 18.637s forward_backward.start -[cold-start] 21.850s forward_backward.done -[cold-start] 22.607s optim_step.done -[cold-start] 22.611s save_weights_and_get_sampling_client.start -[cold-start] 23.184s save_weights_and_get_sampling_client.done -[cold-start] 23.185s sample.start -[cold-start] 130.546s sample.done {'text': ' 4'} -[cold-start] 130.547s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt deleted file mode 100644 index 565ffbe3..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:03:26.774+00:00 diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json deleted file mode 100644 index 8ac2687a..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/events.json +++ /dev/null @@ -1,2025 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:10Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:10Z", - "name": "cold-start-fio.18de9a0ef295773d", - "namespace": "openrl-system", - "resourceVersion": "1792039090760383011", - "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5be63862", - "namespace": "openrl-system", - "resourceVersion": "1792039092515007000", - "uid": "355b59b9-3661-499d-8640-3f40c3c89375" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5ecfa51a", - "namespace": "openrl-system", - "resourceVersion": "1792039092561663021", - "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f61ddd1e4", - "namespace": "openrl-system", - "resourceVersion": "1792039092605935011", - "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:40:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:40:41Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:40:41Z", - "name": "cold-start-fio.18de9a32081cf908", - "namespace": "openrl-system", - "resourceVersion": "1792039241446287009", - "uid": "14a62e42-13ba-4526-a04d-a934702e1163" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:18Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:18Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:18Z", - "name": "cold-start-fio.18de9a3a9867ce9f", - "namespace": "openrl-system", - "resourceVersion": "1792039278208207001", - "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad2bcac91", - "namespace": "openrl-system", - "resourceVersion": "1792039279188751014", - "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad378cd74", - "namespace": "openrl-system", - "resourceVersion": "1792039279219679006", - "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad5aed221", - "namespace": "openrl-system", - "resourceVersion": "1792039279235135013", - "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:11Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:11Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:11Z", - "name": "cold-start-fio.18de9a70cd2c6a68", - "namespace": "openrl-system", - "resourceVersion": "1792039511048479001", - "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "resourceVersion": "1791941114537823021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Stopping container api-server", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", - "namespace": "openrl-system", - "resourceVersion": "1792039545418735013", - "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "namespace": "openrl-system", - "resourceVersion": "1792039545300351007", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", - "namespace": "openrl-system", - "resourceVersion": "1792039545387567001", - "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" - }, - "reason": "SuccessfulDelete", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546048591003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", - "namespace": "openrl-system", - "resourceVersion": "1792039546102351012", - "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:49Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:49Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", - "namespace": "openrl-system", - "resourceVersion": "1792039549712335019", - "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", - "namespace": "openrl-system", - "resourceVersion": "1792039558721215017", - "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", - "namespace": "openrl-system", - "resourceVersion": "1792039558783199013", - "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", - "namespace": "openrl-system", - "resourceVersion": "1792039558862303009", - "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "namespace": "openrl-system", - "resourceVersion": "1792039545940671018", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", - "namespace": "openrl-system", - "resourceVersion": "1792039546068975009", - "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" - }, - "reason": "SuccessfulCreate", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545232655018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78c9668cf7", - "namespace": "openrl-system", - "resourceVersion": "1792039545327119005", - "uid": "7b1433df-3524-478b-aba2-8979950e245e" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545464319018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78ef8b9e1b", - "namespace": "openrl-system", - "resourceVersion": "1792039545956079020", - "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:15Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875490623001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:15Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", - "namespace": "openrl-system", - "resourceVersion": "1792039875609279013", - "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:19Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:19Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", - "namespace": "openrl-system", - "resourceVersion": "1792039879126319008", - "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", - "namespace": "openrl-system", - "resourceVersion": "1792040061645247020", - "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", - "namespace": "openrl-system", - "resourceVersion": "1792040061849391002", - "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", - "namespace": "openrl-system", - "resourceVersion": "1792040062252079023", - "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", - "namespace": "openrl-system", - "resourceVersion": "1792040192780783021", - "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264403887001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", - "namespace": "openrl-system", - "resourceVersion": "1792040264444463021", - "uid": "a85006a2-68a6-4919-b148-08d58b546d07" - }, - "reason": "FailedScheduling", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Warning" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264434463001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", - "namespace": "openrl-system", - "resourceVersion": "1792040264884223002", - "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" - }, - "reason": "NotTriggerScaleUp", - "reportingComponent": "cluster-autoscaler", - "reportingInstance": "", - "source": { - "component": "cluster-autoscaler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264893039001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", - "namespace": "openrl-system", - "resourceVersion": "1792040293171567001", - "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", - "namespace": "openrl-system", - "resourceVersion": "1792040296672127014", - "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", - "namespace": "openrl-system", - "resourceVersion": "1792040296703007017", - "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", - "namespace": "openrl-system", - "resourceVersion": "1792040296889487014", - "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", - "namespace": "openrl-system", - "resourceVersion": "1792040427147711019", - "uid": "cc4da270-def2-456f-9374-372be8c717e4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:38Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498635999001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:38Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:38Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", - "namespace": "openrl-system", - "resourceVersion": "1792040498767471000", - "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", - "namespace": "openrl-system", - "resourceVersion": "1792040502270255017", - "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", - "namespace": "openrl-system", - "resourceVersion": "1792040502296943006", - "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", - "namespace": "openrl-system", - "resourceVersion": "1792040502475407023", - "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:52Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672220015012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:52Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", - "namespace": "openrl-system", - "resourceVersion": "1792039672350591019", - "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:56Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:56Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:56Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", - "namespace": "openrl-system", - "resourceVersion": "1792039676166239015", - "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", - "namespace": "openrl-system", - "resourceVersion": "1792039844457167000", - "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", - "namespace": "openrl-system", - "resourceVersion": "1792039844632591021", - "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:45Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:45Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", - "namespace": "openrl-system", - "resourceVersion": "1792039845170959016", - "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", - "namespace": "openrl-system", - "resourceVersion": "1792040192981183013", - "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:23Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040242926223012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:23Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:23Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", - "namespace": "openrl-system", - "resourceVersion": "1792040243043743024", - "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", - "namespace": "openrl-system", - "resourceVersion": "1792040246545487023", - "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", - "namespace": "openrl-system", - "resourceVersion": "1792040246576895006", - "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", - "namespace": "openrl-system", - "resourceVersion": "1792040246742815002", - "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", - "namespace": "openrl-system", - "resourceVersion": "1792040427285631023", - "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:16Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476805407012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:16Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", - "namespace": "openrl-system", - "resourceVersion": "1792040476942623014", - "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", - "namespace": "openrl-system", - "resourceVersion": "1792040480747055005", - "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", - "namespace": "openrl-system", - "resourceVersion": "1792040480787967019", - "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", - "namespace": "openrl-system", - "resourceVersion": "1792040480963103007", - "uid": "d3158913-c79e-43d4-b468-d748641d49fe" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", - "namespace": "openrl-system", - "resourceVersion": "1792038835732623016", - "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", - "namespace": "openrl-system", - "resourceVersion": "1792038835415695012", - "uid": "918abc9a-1f24-4767-8928-b202482f11ed" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt deleted file mode 100644 index b57d24d4..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/log-orw-lora-qwen-qwen2-5-0-5b-0-sampler.txt +++ /dev/null @@ -1,120 +0,0 @@ -2026-10-15T05:02:05.308858557Z INFO 10-15 05:02:05 [model.py:692] Resolved architecture: Qwen2ForCausalLM -2026-10-15T05:02:05.309341375Z INFO 10-15 05:02:05 [model.py:2030] Using max model len 8192 -2026-10-15T05:02:05.311509464Z INFO 10-15 05:02:05 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=4096. -2026-10-15T05:02:06.441297160Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T05:02:06.722078744Z INFO 10-15 05:02:06 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T05:02:10.058431917Z WARNING 10-15 05:02:10 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T05:02:21.575184973Z (EngineCore pid=161) INFO 10-15 05:02:21 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen2.5-0.5B', speculative_config=None, tokenizer='Qwen/Qwen2.5-0.5B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen2.5-0.5B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T05:02:22.390749815Z (EngineCore pid=161) INFO 10-15 05:02:22 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_da4da587630c4760931dd312e0d64a19 backend=nccl -2026-10-15T05:02:22.398295764Z (EngineCore pid=161) INFO 10-15 05:02:22 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T05:02:22.398324742Z (EngineCore pid=161) INFO 10-15 05:02:22 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T05:02:23.325863393Z (EngineCore pid=161) INFO 10-15 05:02:23 [model_runner.py:396] Loading model from scratch... -2026-10-15T05:02:24.203367263Z (EngineCore pid=161) INFO 10-15 05:02:24 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T05:02:24.203834642Z (EngineCore pid=161) INFO 10-15 05:02:24 [flash_attn.py:1116] Using FlashAttention version 2 -2026-10-15T05:02:24.320213947Z (EngineCore pid=161) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T05:02:24.320556376Z (EngineCore pid=161) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T05:02:24.679861647Z (EngineCore pid=161) INFO 10-15 05:02:24 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 0.92 GiB. Available RAM: 22.61 GiB. -2026-10-15T05:02:24.679896304Z (EngineCore pid=161) INFO 10-15 05:02:24 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T05:02:24.681277221Z (EngineCore pid=161) -Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T05:01:27.867911082Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T05:01:27.867937281Z -2026-10-15T05:01:27.989644961Z Loading base model Qwen/Qwen2.5-0.5B to cuda:0... -2026-10-15T05:01:28.104973244Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T05:01:28.918184929Z [prefetch] reading 1 file(s), 0.92 GiB, from nfs into the page cache with 8 threads -2026-10-15T05:01:29.629438937Z [prefetch] done: 0.92 GiB in 0.7 s (1325 MiB/s) -2026-10-15T05:01:30.447285388Z -Loading weights: 0%| | 0/290 [00:00 9d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 true -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 true diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json deleted file mode 100644 index 5ee0198a..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "generateName": "open-rl-api-server-6d86b498f5-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "6d86b498f5", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - } - ], - "resourceVersion": "1792039570790271003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2ns79", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:59Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:45:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.11", - "podIPs": [ - { - "ip": "10.32.3.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:45:46Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:38Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-sampler", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "uid": "7b9b59dc-9b47-457a-ae6e-445216e97f77" - } - ], - "resourceVersion": "1792040502744719001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "dc12971c-f814-48f0-989f-61dcbd87544b" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2dmx2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2dmx2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:42Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "25Gi" - }, - "containerID": "containerd://9ab93d789300dabc0ddb29eecd8d52752a8620fb49c116df96da13be6a371073", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:01:42Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2dmx2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.0.22", - "podIPs": [ - { - "ip": "10.32.0.22" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:01:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:16Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", - "timeslice.io/job-id": "lora-qwen-qwen2-5-0-5b-0-trainer", - "timeslice.io/owner": "qwen-qwen2-5-0-5b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "uid": "2e5fb99a-5dca-43b7-9297-8a45c4fc96e0" - } - ], - "resourceVersion": "1792040481706719012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "9ba5505c-ce23-4a59-ba01-41cad9093eca" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-g89d5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-g89d5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:21Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:16Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:21Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:21Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:01:16Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "21Gi" - }, - "containerID": "containerd://996f69bb4e5188384112befeb9a6e19076f01e6d3416b8e194129863978b175f", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:01:20Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-g89d5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.13.28", - "podIPs": [ - { - "ip": "10.32.13.28" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:01:16Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log deleted file mode 100644 index 62cf050e..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/redis-ready.log +++ /dev/null @@ -1,153 +0,0 @@ -2026-10-15T05:01:12.427+00:00 2026-10-15T05:01:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:13.431+00:00 2026-10-15T05:01:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:14.437+00:00 2026-10-15T05:01:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:15.442+00:00 2026-10-15T05:01:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:16.446+00:00 2026-10-15T05:01:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:17.451+00:00 2026-10-15T05:01:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:18.458+00:00 2026-10-15T05:01:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:19.467+00:00 2026-10-15T05:01:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:20.468+00:00 2026-10-15T05:01:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:21.477+00:00 2026-10-15T05:01:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:22.478+00:00 2026-10-15T05:01:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:23.483+00:00 2026-10-15T05:01:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:24.489+00:00 2026-10-15T05:01:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:25.494+00:00 2026-10-15T05:01:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:26.508+00:00 2026-10-15T05:01:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:27.503+00:00 2026-10-15T05:01:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:28.508+00:00 2026-10-15T05:01:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:29.516+00:00 2026-10-15T05:01:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:30.519+00:00 2026-10-15T05:01:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:31.524+00:00 2026-10-15T05:01:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:32.529+00:00 2026-10-15T05:01:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:33.534+00:00 2026-10-15T05:01:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:34.539+00:00 2026-10-15T05:01:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:35.545+00:00 2026-10-15T05:01:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:36.550+00:00 2026-10-15T05:01:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:37.555+00:00 2026-10-15T05:01:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:38.560+00:00 2026-10-15T05:01:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:39.565+00:00 2026-10-15T05:01:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:40.574+00:00 2026-10-15T05:01:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:41.576+00:00 2026-10-15T05:01:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:42.581+00:00 2026-10-15T05:01:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:43.588+00:00 2026-10-15T05:01:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:44.592+00:00 2026-10-15T05:01:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:45.598+00:00 2026-10-15T05:01:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:46.602+00:00 2026-10-15T05:01:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:47.608+00:00 2026-10-15T05:01:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:48.613+00:00 2026-10-15T05:01:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:49.618+00:00 2026-10-15T05:01:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:50.624+00:00 2026-10-15T05:01:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:51.628+00:00 2026-10-15T05:01:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:52.644+00:00 2026-10-15T05:01:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:53.654+00:00 2026-10-15T05:01:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:54.650+00:00 2026-10-15T05:01:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:55.648+00:00 2026-10-15T05:01:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:56.653+00:00 2026-10-15T05:01:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:57.659+00:00 2026-10-15T05:01:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:58.673+00:00 2026-10-15T05:01:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:01:59.674+00:00 2026-10-15T05:01:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:00.675+00:00 2026-10-15T05:02:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:01.679+00:00 2026-10-15T05:02:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:02.687+00:00 2026-10-15T05:02:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:03.690+00:00 2026-10-15T05:02:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:04.695+00:00 2026-10-15T05:02:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:05.700+00:00 2026-10-15T05:02:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:06.706+00:00 2026-10-15T05:02:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:07.710+00:00 2026-10-15T05:02:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:08.720+00:00 2026-10-15T05:02:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:09.721+00:00 2026-10-15T05:02:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:10.726+00:00 2026-10-15T05:02:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:11.731+00:00 2026-10-15T05:02:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:12.736+00:00 2026-10-15T05:02:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:13.767+00:00 2026-10-15T05:02:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:14.746+00:00 2026-10-15T05:02:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:15.752+00:00 2026-10-15T05:02:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:16.758+00:00 2026-10-15T05:02:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:17.764+00:00 2026-10-15T05:02:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:18.780+00:00 2026-10-15T05:02:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:19.773+00:00 2026-10-15T05:02:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:20.777+00:00 2026-10-15T05:02:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:21.783+00:00 2026-10-15T05:02:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:22.790+00:00 2026-10-15T05:02:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:23.794+00:00 2026-10-15T05:02:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:24.817+00:00 2026-10-15T05:02:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:25.805+00:00 2026-10-15T05:02:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:26.809+00:00 2026-10-15T05:02:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:27.815+00:00 2026-10-15T05:02:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:28.820+00:00 2026-10-15T05:02:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:29.825+00:00 2026-10-15T05:02:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:30.831+00:00 2026-10-15T05:02:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:31.836+00:00 2026-10-15T05:02:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:32.845+00:00 2026-10-15T05:02:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:33.847+00:00 2026-10-15T05:02:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:34.852+00:00 2026-10-15T05:02:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:35.865+00:00 2026-10-15T05:02:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:36.874+00:00 2026-10-15T05:02:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:37.867+00:00 2026-10-15T05:02:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:38.875+00:00 2026-10-15T05:02:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:39.878+00:00 2026-10-15T05:02:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:40.883+00:00 2026-10-15T05:02:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:41.888+00:00 2026-10-15T05:02:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:42.893+00:00 2026-10-15T05:02:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:43.898+00:00 2026-10-15T05:02:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:44.905+00:00 2026-10-15T05:02:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:45.916+00:00 2026-10-15T05:02:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:46.915+00:00 2026-10-15T05:02:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:47.919+00:00 2026-10-15T05:02:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:48.925+00:00 2026-10-15T05:02:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:49.930+00:00 2026-10-15T05:02:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:50.935+00:00 2026-10-15T05:02:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:51.939+00:00 2026-10-15T05:02:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:52.944+00:00 2026-10-15T05:02:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:53.949+00:00 2026-10-15T05:02:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:54.955+00:00 2026-10-15T05:02:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:55.959+00:00 2026-10-15T05:02:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:56.966+00:00 2026-10-15T05:02:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:57.973+00:00 2026-10-15T05:02:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:58.975+00:00 2026-10-15T05:02:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:02:59.981+00:00 2026-10-15T05:02:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:00.987+00:00 2026-10-15T05:03:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:01.991+00:00 2026-10-15T05:03:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:02.997+00:00 2026-10-15T05:03:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:04.005+00:00 2026-10-15T05:03:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:05.011+00:00 2026-10-15T05:03:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:06.012+00:00 2026-10-15T05:03:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:07.020+00:00 2026-10-15T05:03:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:08.022+00:00 2026-10-15T05:03:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:09.027+00:00 2026-10-15T05:03:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:10.034+00:00 2026-10-15T05:03:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:11.038+00:00 2026-10-15T05:03:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:12.043+00:00 2026-10-15T05:03:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:13.048+00:00 2026-10-15T05:03:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:14.055+00:00 2026-10-15T05:03:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:15.059+00:00 2026-10-15T05:03:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:16.067+00:00 2026-10-15T05:03:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:17.069+00:00 2026-10-15T05:03:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:18.078+00:00 2026-10-15T05:03:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:19.079+00:00 2026-10-15T05:03:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:20.085+00:00 2026-10-15T05:03:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:21.093+00:00 2026-10-15T05:03:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:22.097+00:00 2026-10-15T05:03:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:23.100+00:00 2026-10-15T05:03:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:24.105+00:00 2026-10-15T05:03:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:25.110+00:00 2026-10-15T05:03:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:26.123+00:00 2026-10-15T05:03:26Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:26.123+00:00 2026-10-15T05:03:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:27.142+00:00 2026-10-15T05:03:27Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:27.142+00:00 2026-10-15T05:03:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:28.126+00:00 2026-10-15T05:03:28Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:28.126+00:00 2026-10-15T05:03:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:29.130+00:00 2026-10-15T05:03:29Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:29.130+00:00 2026-10-15T05:03:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:30.157+00:00 2026-10-15T05:03:30Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:30.157+00:00 2026-10-15T05:03:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:31.216+00:00 2026-10-15T05:03:31Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:31.216+00:00 2026-10-15T05:03:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:32.160+00:00 2026-10-15T05:03:32Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:32.160+00:00 2026-10-15T05:03:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:33.185+00:00 2026-10-15T05:03:33Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:33.186+00:00 2026-10-15T05:03:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:34.183+00:00 2026-10-15T05:03:34Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:34.183+00:00 2026-10-15T05:03:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:35.226+00:00 2026-10-15T05:03:35Z open_rl:sampler_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:03:35.226+00:00 2026-10-15T05:03:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log deleted file mode 100644 index 0a19fdc3..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T05:01:16.606+00:00 2026-10-15T05:01:16.624535671Z {"level":"info","ts":"2026-10-15T05:01:16Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-trainer","reconcileID":"9572772f-a156-403a-b664-332350bac2f5","claim":"claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a","worker":"lora-qwen-qwen2-5-0-5b-0-trainer","tiers":"t1x23"} -2026-10-15T05:01:38.487+00:00 2026-10-15T05:01:38.505401957Z {"level":"info","ts":"2026-10-15T05:01:38Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen2-5-0-5b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen2-5-0-5b-0-sampler","reconcileID":"1a58b154-9236-4c92-aab9-519ccf1bd949","claim":"claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc","worker":"lora-qwen-qwen2-5-0-5b-0-sampler","tiers":"t1x23"} diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json deleted file mode 100644 index 05335083..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/timeline.json +++ /dev/null @@ -1,267 +0,0 @@ -{ - "case": "after-B2-0.5b-l4-cached", - "collected": "2026-10-15T05:03:35.925+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 05:01:15.634000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen2.5-0.5B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 05:01:15.634000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 05:01:31.419000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "840431b4-6617-4074-8848-cf95526c9aec" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 05:01:34.271000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 05:01:37.484000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 05:01:38.241000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 05:01:38.245000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 05:01:38.818000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 05:01:38.819000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 05:03:26.180000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4" - }, - "client.end": { - "ts": "2026-10-15 05:03:26.181000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen2-5-0-5b-0-sampler", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 05:01:38+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:01:38.028000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 05:01:38.028000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 05:01:39.789000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 05:01:42+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 05:01:38+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 05:01:38+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 05:01:42+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 05:01:42+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 05:01:38+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 05:01:42+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 05:01:38+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 05:01:42+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "sampler.event_Created": { - "ts": "2026-10-15 05:01:42+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 05:01:42+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 05:02:05.308858+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 05:02:21.575184+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:21 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 05:02:23.325863+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:23 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 05:02:24.679896+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:24 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 05:02:25.907445+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:25 [default_loader.py:430] Loading weights took 1.23 seconds", - "self_reported_s": 1.23 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 05:02:26.718502+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:26 [model_runner.py:428] Model loading took 1.47 GiB memory and 3.717496 seconds", - "self_reported_s": 3.717496 - }, - "sampler.compile_done": { - "ts": "2026-10-15 05:02:57.251696+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:02:57 [monitor.py:53] torch.compile took 28.32 s in total", - "self_reported_s": 28.32 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 05:03:12.777714+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:03:12 [model_runner.py:1066] Graph capturing finished in 11 secs, took 0.23 GiB", - "self_reported_s": 11.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 05:03:13.327483+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:03:13 [gpu_worker.py:640] Available KV cache memory: 17.33 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 05:03:24.097775+00:00", - "source": "container log: (EngineCore pid=161) INFO 10-15 05:03:24 [core.py:372] init engine (profile, create kv cache, warmup model) took 57.37 s", - "self_reported_s": 57.37 - }, - "sampler.ready_key": { - "ts": "2026-10-15 05:03:26+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - }, - "trainer": { - "workload": "lora-qwen-qwen2-5-0-5b-0-trainer", - "model_id": "Qwen/Qwen2.5-0.5B", - "node": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "pod": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 05:01:16+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:01:16.282000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 05:01:16.282000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 05:01:17.948000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 05:01:22+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 05:01:16+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 05:01:16+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 05:01:21+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 05:01:21+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 05:01:16+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 05:01:20+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 05:01:16+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 05:01:20+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "trainer.event_Created": { - "ts": "2026-10-15 05:01:20+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 05:01:20+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 05:01:27.847766+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 05:01:27.989644+00:00", - "source": "container log: Loading base model Qwen/Qwen2.5-0.5B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 05:01:30.531288+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 05:01:30.533708+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen2.5-0.5B active_set=Qwen/Qwen2.5-0.5B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 05:01:12+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt deleted file mode 100644 index b19e78b5..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:01:11.324+00:00 diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt deleted file mode 100644 index a9917932..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json b/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json deleted file mode 100644 index a4b62ba3..00000000 --- a/runs/2026-10-15_cold_start/after-B2-0.5b-l4-cached/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:38Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040502944559001", - "uid": "7b9b59dc-9b47-457a-ae6e-445216e97f77" - }, - "spec": { - "accelerator": { - "memory": "4Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "3589169104" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "25Gi" - }, - "requests": { - "memory": "25Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "dc12971c-f814-48f0-989f-61dcbd87544b", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:01:42Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-sampler-7b9b59dc", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "4Gi", - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:16Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040482420783015", - "uid": "2e5fb99a-5dca-43b7-9297-8a45c4fc96e0" - }, - "spec": { - "accelerator": { - "memory": "5Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen2.5-0.5B", - "ownerID": "qwen-qwen2-5-0-5b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen2.5-0.5B", - "--active-tenant-set-id", - "Qwen/Qwen2.5-0.5B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen2.5-0.5B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "5282967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen2-5-0-5b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "21Gi" - }, - "requests": { - "memory": "21Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "9ba5505c-ce23-4a59-ba01-41cad9093eca", - "claimName": "claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:01:22Z", - "message": "worker is running on claim-lora-qwen-qwen2-5-0-5b-0-trainer-2e5fb99a", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "5Gi", - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen2-5-0-5b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log deleted file mode 100644 index afa62d96..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold.run.log +++ /dev/null @@ -1,35 +0,0 @@ -watching; recorders [16726, 16727, 16729, 16731, 16733, 16728, 16730, 16732] write into runs/2026-10-15_cold_start/after-C1-8b-h100-cold -collected 2 worker log(s) into runs/2026-10-15_cold_start/after-C1-8b-h100-cold -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.9 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 90.0 | -| container create | trainer.event_Pulled | trainer.container_started | 0.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.1 | -| weights load | trainer.load_start | trainer.load_done | 13.0 | -| ready key | trainer.load_done | trainer.ready_key | -0.1 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 111.5 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.8 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.4 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 90.0 | -| container create | sampler.event_Pulled | sampler.container_started | 0.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 7.5 | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 17.5 | -| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 14.0 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.5 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.9 | -| ready key | sampler.engine_init_done | sampler.ready_key | 0.6 | -| client: first sample blocked | client.sample.start | client.sample.done | 207.9 | - diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log deleted file mode 100644 index 4097aea9..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/api-server.log +++ /dev/null @@ -1,56 +0,0 @@ -2026-10-15T05:04:58.373+00:00 2026-10-15T05:04:58.392830523Z INFO: 127.0.0.1:49988 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T05:04:58.599+00:00 2026-10-15T05:04:58.617434754Z INFO: 127.0.0.1:49998 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T05:04:58.671+00:00 2026-10-15T05:04:58.691762251Z INFO: 127.0.0.1:49998 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T05:04:58.912+00:00 2026-10-15T05:04:58.933022406Z [API_SERVER] enqueued op=create_model request_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:04:58.912+00:00 2026-10-15T05:04:58.934353835Z INFO: 127.0.0.1:50002 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T05:05:00.726+00:00 2026-10-15T05:05:00.747482128Z INFO: 10.32.3.1:57206 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:05:08.868+00:00 2026-10-15T05:05:08.886137925Z INFO: 127.0.0.1:46950 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:05:10.727+00:00 2026-10-15T05:05:10.747267963Z INFO: 10.32.3.1:60820 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:05:20.728+00:00 2026-10-15T05:05:20.746735897Z INFO: 10.32.3.1:45172 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:05:30.730+00:00 2026-10-15T05:05:30.747739235Z INFO: 10.32.3.1:42562 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:05:32.132+00:00 2026-10-15T05:05:32.152552067Z [API_SERVER] No live session uses qwen-qwen2-5-0-5b; tearing its workers down -2026-10-15T05:05:40.729+00:00 2026-10-15T05:05:40.748089335Z INFO: 10.32.3.1:39376 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:05:49.048+00:00 2026-10-15T05:05:49.066216479Z INFO: 127.0.0.1:38418 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:05:50.725+00:00 2026-10-15T05:05:50.747579061Z INFO: 10.32.3.1:46244 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:06:00.730+00:00 2026-10-15T05:06:00.747988598Z INFO: 10.32.3.1:44906 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:06:10.731+00:00 2026-10-15T05:06:10.748609740Z INFO: 10.32.3.1:46228 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:06:20.729+00:00 2026-10-15T05:06:20.747688850Z INFO: 10.32.3.1:60052 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:06:30.728+00:00 2026-10-15T05:06:30.747502924Z INFO: 10.32.3.1:43338 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:06:39.279+00:00 2026-10-15T05:06:39.296775538Z INFO: 127.0.0.1:47514 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:06:40.724+00:00 2026-10-15T05:06:40.747079682Z INFO: 10.32.3.1:54366 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:06:49.774+00:00 2026-10-15T05:06:49.794105686Z INFO: 127.0.0.1:40780 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T05:06:50.724+00:00 2026-10-15T05:06:50.746682232Z INFO: 10.32.3.1:54610 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:06:52.856+00:00 2026-10-15T05:06:52.877671436Z [API_SERVER] enqueued op=forward_backward request_id=89817a3b-47fd-4fe7-ac4f-96f85e67f00d model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:06:52.857+00:00 2026-10-15T05:06:52.879485463Z INFO: 127.0.0.1:40780 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T05:06:55.673+00:00 2026-10-15T05:06:55.694412936Z [API_SERVER] enqueued op=optim_step request_id=960757cf-ac36-466e-8ac6-4739735165f9 model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:06:55.676+00:00 2026-10-15T05:06:55.695623233Z INFO: 127.0.0.1:40780 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T05:06:57.720+00:00 2026-10-15T05:06:57.740198942Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=8e6d158e-97d2-4b55-969a-3c06dfb5aaf1 model_id=048101ca-2c60-4c63-afa2-2ae75e83f1c4 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:06:57.720+00:00 2026-10-15T05:06:57.741174270Z INFO: 127.0.0.1:40780 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T05:06:59.833+00:00 2026-10-15T05:06:59.854047429Z INFO: 127.0.0.1:38466 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T05:07:00.725+00:00 2026-10-15T05:07:00.748423093Z INFO: 10.32.3.1:34800 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:07:10.729+00:00 2026-10-15T05:07:10.747610422Z INFO: 10.32.3.1:37484 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:07:20.727+00:00 2026-10-15T05:07:20.747378396Z INFO: 10.32.3.1:46200 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:07:30.729+00:00 2026-10-15T05:07:30.746887455Z INFO: 10.32.3.1:32866 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:07:40.729+00:00 2026-10-15T05:07:40.747803363Z INFO: 10.32.3.1:34618 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:07:49.470+00:00 2026-10-15T05:07:49.486856420Z INFO: 127.0.0.1:35182 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:07:50.727+00:00 2026-10-15T05:07:50.749079833Z INFO: 10.32.3.1:44656 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:08:00.731+00:00 2026-10-15T05:08:00.746828463Z INFO: 10.32.3.1:52534 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:08:10.729+00:00 2026-10-15T05:08:10.747693054Z INFO: 10.32.3.1:45248 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:08:20.741+00:00 2026-10-15T05:08:20.747102416Z INFO: 10.32.3.1:35372 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:08:30.727+00:00 2026-10-15T05:08:30.747138544Z INFO: 10.32.3.1:46088 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:08:39.651+00:00 2026-10-15T05:08:39.669833714Z INFO: 127.0.0.1:42494 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:08:40.820+00:00 2026-10-15T05:08:40.748143407Z INFO: 10.32.3.1:60424 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:08:50.737+00:00 2026-10-15T05:08:50.748661295Z INFO: 10.32.3.1:34518 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:09:00.727+00:00 2026-10-15T05:09:00.747876484Z INFO: 10.32.3.1:50858 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:09:10.728+00:00 2026-10-15T05:09:10.748210445Z INFO: 10.32.3.1:35368 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:09:19.829+00:00 2026-10-15T05:09:19.849115515Z INFO: 127.0.0.1:47558 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:09:20.723+00:00 2026-10-15T05:09:20.746874486Z INFO: 10.32.3.1:39832 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:09:30.729+00:00 2026-10-15T05:09:30.747161062Z INFO: 10.32.3.1:47028 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:09:40.727+00:00 2026-10-15T05:09:40.747722414Z INFO: 10.32.3.1:41054 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:09:50.725+00:00 2026-10-15T05:09:50.746880809Z INFO: 10.32.3.1:57554 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:09:58.736+00:00 2026-10-15T05:09:58.758631423Z INFO: 127.0.0.1:49698 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T05:10:00.726+00:00 2026-10-15T05:10:00.747754462Z INFO: 10.32.3.1:51350 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:10:10.039+00:00 2026-10-15T05:10:10.061264177Z INFO: 127.0.0.1:42950 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:10:10.723+00:00 2026-10-15T05:10:10.747463025Z INFO: 10.32.3.1:46708 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:10:20.725+00:00 2026-10-15T05:10:20.746995461Z INFO: 10.32.3.1:50678 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:10:30.725+00:00 2026-10-15T05:10:30.747017967Z INFO: 10.32.3.1:36428 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl deleted file mode 100644 index a66ee2ba..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T05:04:58.138+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T05:04:58.139+00:00", "t": 0.0, "phase": "create_model.start"} -{"ts": "2026-10-15T05:06:49.594+00:00", "t": 111.456, "phase": "create_model.done", "model_id": "048101ca-2c60-4c63-afa2-2ae75e83f1c4"} -{"ts": "2026-10-15T05:06:52.784+00:00", "t": 114.646, "phase": "forward_backward.start"} -{"ts": "2026-10-15T05:06:55.595+00:00", "t": 117.457, "phase": "forward_backward.done"} -{"ts": "2026-10-15T05:06:57.617+00:00", "t": 119.479, "phase": "optim_step.done"} -{"ts": "2026-10-15T05:06:57.618+00:00", "t": 119.48, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T05:06:59.533+00:00", "t": 121.395, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T05:06:59.534+00:00", "t": 121.395, "phase": "sample.start"} -{"ts": "2026-10-15T05:10:27.431+00:00", "t": 329.293, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} -{"ts": "2026-10-15T05:10:27.434+00:00", "t": 329.296, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log deleted file mode 100644 index 5a4ebab2..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.000s create_model.start -[cold-start] 111.456s create_model.done {'model_id': '048101ca-2c60-4c63-afa2-2ae75e83f1c4'} -[cold-start] 114.646s forward_backward.start -[cold-start] 117.457s forward_backward.done -[cold-start] 119.479s optim_step.done -[cold-start] 119.480s save_weights_and_get_sampling_client.start -[cold-start] 121.395s save_weights_and_get_sampling_client.done -[cold-start] 121.395s sample.start -[cold-start] 329.293s sample.done {'text': ' 4\n\nQuestion: What is '} -[cold-start] 329.296s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt deleted file mode 100644 index 8b3c4311..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:10:28.035+00:00 diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json deleted file mode 100644 index 89cb0ffe..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/events.json +++ /dev/null @@ -1,2417 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:10Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:10Z", - "name": "cold-start-fio.18de9a0ef295773d", - "namespace": "openrl-system", - "resourceVersion": "1792039090760383011", - "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5be63862", - "namespace": "openrl-system", - "resourceVersion": "1792039092515007000", - "uid": "355b59b9-3661-499d-8640-3f40c3c89375" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5ecfa51a", - "namespace": "openrl-system", - "resourceVersion": "1792039092561663021", - "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f61ddd1e4", - "namespace": "openrl-system", - "resourceVersion": "1792039092605935011", - "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:40:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:40:41Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:40:41Z", - "name": "cold-start-fio.18de9a32081cf908", - "namespace": "openrl-system", - "resourceVersion": "1792039241446287009", - "uid": "14a62e42-13ba-4526-a04d-a934702e1163" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:18Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:18Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:18Z", - "name": "cold-start-fio.18de9a3a9867ce9f", - "namespace": "openrl-system", - "resourceVersion": "1792039278208207001", - "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad2bcac91", - "namespace": "openrl-system", - "resourceVersion": "1792039279188751014", - "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad378cd74", - "namespace": "openrl-system", - "resourceVersion": "1792039279219679006", - "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad5aed221", - "namespace": "openrl-system", - "resourceVersion": "1792039279235135013", - "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:11Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:11Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:11Z", - "name": "cold-start-fio.18de9a70cd2c6a68", - "namespace": "openrl-system", - "resourceVersion": "1792039511048479001", - "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "resourceVersion": "1791941114537823021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Stopping container api-server", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", - "namespace": "openrl-system", - "resourceVersion": "1792039545418735013", - "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "namespace": "openrl-system", - "resourceVersion": "1792039545300351007", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", - "namespace": "openrl-system", - "resourceVersion": "1792039545387567001", - "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" - }, - "reason": "SuccessfulDelete", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546048591003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", - "namespace": "openrl-system", - "resourceVersion": "1792039546102351012", - "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:49Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:49Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", - "namespace": "openrl-system", - "resourceVersion": "1792039549712335019", - "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", - "namespace": "openrl-system", - "resourceVersion": "1792039558721215017", - "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", - "namespace": "openrl-system", - "resourceVersion": "1792039558783199013", - "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", - "namespace": "openrl-system", - "resourceVersion": "1792039558862303009", - "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "namespace": "openrl-system", - "resourceVersion": "1792039545940671018", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", - "namespace": "openrl-system", - "resourceVersion": "1792039546068975009", - "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" - }, - "reason": "SuccessfulCreate", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545232655018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78c9668cf7", - "namespace": "openrl-system", - "resourceVersion": "1792039545327119005", - "uid": "7b1433df-3524-478b-aba2-8979950e245e" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545464319018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78ef8b9e1b", - "namespace": "openrl-system", - "resourceVersion": "1792039545956079020", - "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:15Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875490623001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:15Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", - "namespace": "openrl-system", - "resourceVersion": "1792039875609279013", - "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:19Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:19Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", - "namespace": "openrl-system", - "resourceVersion": "1792039879126319008", - "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", - "namespace": "openrl-system", - "resourceVersion": "1792040061645247020", - "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", - "namespace": "openrl-system", - "resourceVersion": "1792040061849391002", - "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", - "namespace": "openrl-system", - "resourceVersion": "1792040062252079023", - "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", - "namespace": "openrl-system", - "resourceVersion": "1792040192780783021", - "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264403887001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", - "namespace": "openrl-system", - "resourceVersion": "1792040264444463021", - "uid": "a85006a2-68a6-4919-b148-08d58b546d07" - }, - "reason": "FailedScheduling", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Warning" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264434463001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", - "namespace": "openrl-system", - "resourceVersion": "1792040264884223002", - "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" - }, - "reason": "NotTriggerScaleUp", - "reportingComponent": "cluster-autoscaler", - "reportingInstance": "", - "source": { - "component": "cluster-autoscaler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264893039001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", - "namespace": "openrl-system", - "resourceVersion": "1792040293171567001", - "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", - "namespace": "openrl-system", - "resourceVersion": "1792040296672127014", - "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", - "namespace": "openrl-system", - "resourceVersion": "1792040296703007017", - "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", - "namespace": "openrl-system", - "resourceVersion": "1792040296889487014", - "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", - "namespace": "openrl-system", - "resourceVersion": "1792040427147711019", - "uid": "cc4da270-def2-456f-9374-372be8c717e4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:38Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498635999001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:38Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:38Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", - "namespace": "openrl-system", - "resourceVersion": "1792040498767471000", - "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", - "namespace": "openrl-system", - "resourceVersion": "1792040502270255017", - "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", - "namespace": "openrl-system", - "resourceVersion": "1792040502296943006", - "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", - "namespace": "openrl-system", - "resourceVersion": "1792040502475407023", - "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:09Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:09Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:09Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", - "namespace": "openrl-system", - "resourceVersion": "1792040649924575012", - "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:52Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672220015012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:52Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", - "namespace": "openrl-system", - "resourceVersion": "1792039672350591019", - "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:56Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:56Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:56Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", - "namespace": "openrl-system", - "resourceVersion": "1792039676166239015", - "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", - "namespace": "openrl-system", - "resourceVersion": "1792039844457167000", - "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", - "namespace": "openrl-system", - "resourceVersion": "1792039844632591021", - "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:45Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:45Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", - "namespace": "openrl-system", - "resourceVersion": "1792039845170959016", - "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", - "namespace": "openrl-system", - "resourceVersion": "1792040192981183013", - "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:23Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040242926223012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:23Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:23Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", - "namespace": "openrl-system", - "resourceVersion": "1792040243043743024", - "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", - "namespace": "openrl-system", - "resourceVersion": "1792040246545487023", - "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", - "namespace": "openrl-system", - "resourceVersion": "1792040246576895006", - "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", - "namespace": "openrl-system", - "resourceVersion": "1792040246742815002", - "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", - "namespace": "openrl-system", - "resourceVersion": "1792040427285631023", - "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:16Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476805407012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:16Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", - "namespace": "openrl-system", - "resourceVersion": "1792040476942623014", - "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", - "namespace": "openrl-system", - "resourceVersion": "1792040480747055005", - "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", - "namespace": "openrl-system", - "resourceVersion": "1792040480787967019", - "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", - "namespace": "openrl-system", - "resourceVersion": "1792040480963103007", - "uid": "d3158913-c79e-43d4-b468-d748641d49fe" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:10Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:10Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", - "namespace": "openrl-system", - "resourceVersion": "1792040650191935010", - "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", - "namespace": "openrl-system", - "resourceVersion": "1792038835732623016", - "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:58Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818213471023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:58Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:58Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", - "namespace": "openrl-system", - "resourceVersion": "1792040818411407005", - "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:07:01Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:07:01Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:07:01Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", - "namespace": "openrl-system", - "resourceVersion": "1792040821861695012", - "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", - "namespace": "openrl-system", - "resourceVersion": "1792040908347887024", - "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", - "namespace": "openrl-system", - "resourceVersion": "1792040908367439012", - "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", - "namespace": "openrl-system", - "resourceVersion": "1792040908418671004", - "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", - "namespace": "openrl-system", - "resourceVersion": "1792038835415695012", - "uid": "918abc9a-1f24-4767-8928-b202482f11ed" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:59Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699253647006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:59Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:59Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", - "namespace": "openrl-system", - "resourceVersion": "1792040699411615020", - "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:05:02Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:05:02Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:05:02Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", - "namespace": "openrl-system", - "resourceVersion": "1792040702849391020", - "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", - "namespace": "openrl-system", - "resourceVersion": "1792040789347727015", - "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", - "namespace": "openrl-system", - "resourceVersion": "1792040789380863020", - "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", - "namespace": "openrl-system", - "resourceVersion": "1792040789417615006", - "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt deleted file mode 100644 index c45083ae..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/log-orw-lora-qwen-qwen3-8b-0-sampler.txt +++ /dev/null @@ -1,223 +0,0 @@ -2026-10-15T05:08:35.508944913Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T05:08:43.408014705Z INFO 10-15 05:08:43 [model.py:692] Resolved architecture: Qwen3ForCausalLM -2026-10-15T05:08:43.408258811Z INFO 10-15 05:08:43 [model.py:2030] Using max model len 8192 -2026-10-15T05:08:43.409800014Z INFO 10-15 05:08:43 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. -2026-10-15T05:08:43.410611953Z INFO 10-15 05:08:43 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T05:08:45.859619092Z WARNING 10-15 05:08:45 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T05:08:52.969476089Z (EngineCore pid=169) INFO 10-15 05:08:52 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T05:08:53.310496973Z (EngineCore pid=169) INFO 10-15 05:08:53 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_fbe13a8f695f4e8c9b96a62df6e018fb backend=nccl -2026-10-15T05:08:53.314764503Z (EngineCore pid=169) INFO 10-15 05:08:53 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T05:08:53.314781888Z (EngineCore pid=169) INFO 10-15 05:08:53 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T05:08:54.079567248Z (EngineCore pid=169) INFO 10-15 05:08:54 [model_runner.py:396] Loading model from scratch... -2026-10-15T05:08:54.655291043Z (EngineCore pid=169) INFO 10-15 05:08:54 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T05:08:54.655331658Z (EngineCore pid=169) INFO 10-15 05:08:54 [flash_attn.py:1116] Using FlashAttention version 3 -2026-10-15T05:08:54.768638229Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T05:08:54.768672205Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T05:08:55.068635806Z (EngineCore pid=169) INFO 10-15 05:08:55 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. -2026-10-15T05:08:55.068664846Z (EngineCore pid=169) INFO 10-15 05:08:55 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T05:08:55.069420485Z (EngineCore pid=169) -Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T05:06:33.868612492Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T05:06:33.868619724Z -2026-10-15T05:06:34.074918680Z Loading base model Qwen/Qwen3-8B to cuda:0... -2026-10-15T05:06:34.296341094Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T05:06:35.101730406Z [prefetch] reading 5 file(s), 15.26 GiB, from nfs into the page cache with 8 threads -2026-10-15T05:06:44.477515651Z [prefetch] done: 15.26 GiB in 9.4 s (1666 MiB/s) -2026-10-15T05:06:46.952210680Z -Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json deleted file mode 100644 index b2c3f8d6..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "generateName": "open-rl-api-server-6d86b498f5-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "6d86b498f5", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - } - ], - "resourceVersion": "1792039570790271003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2ns79", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:59Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:45:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.11", - "podIPs": [ - { - "ip": "10.32.3.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:45:46Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:58Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-sampler", - "uid": "3920e58e-f415-4c96-8485-77a8cd538872" - } - ], - "resourceVersion": "1792040908564591023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-sampler-3920e58e" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "44791173-6b7c-4d22-9a8b-46ef83190384" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-vnbl2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-vnbl2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:08:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:06:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:08:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:08:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:06:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "40Gi" - }, - "containerID": "containerd://3e4a6241afecb5338cb06427e1604c19bcecb3758c35b1a359aa2774bbab9c1b", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:08:28Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-vnbl2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.1.9", - "podIPs": [ - { - "ip": "10.32.1.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:06:58Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:59Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-trainer", - "uid": "125fac70-8359-4f66-bcb5-20df4c1e1764" - } - ], - "resourceVersion": "1792040789855759006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-trainer-125fac70" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "371ea4b9-ae90-4e15-a7b1-b34802637e23" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5vhhk", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-5vhhk", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:06:29Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:04:59Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:06:29Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:06:29Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:04:59Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "36Gi" - }, - "containerID": "containerd://e6e0c04018d8bd44ea79dcdecc69553e9707fd65313f4196c6ac54d3260924ce", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:06:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5vhhk", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.4.10", - "podIPs": [ - { - "ip": "10.32.4.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:04:59Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log deleted file mode 100644 index fef5c86f..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/redis-ready.log +++ /dev/null @@ -1,575 +0,0 @@ -2026-10-15T05:04:55.021+00:00 2026-10-15T05:04:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:04:56.026+00:00 2026-10-15T05:04:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:04:57.031+00:00 2026-10-15T05:04:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:04:58.037+00:00 2026-10-15T05:04:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:04:59.042+00:00 2026-10-15T05:04:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:00.047+00:00 2026-10-15T05:05:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:01.055+00:00 2026-10-15T05:05:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:02.059+00:00 2026-10-15T05:05:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:03.066+00:00 2026-10-15T05:05:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:04.069+00:00 2026-10-15T05:05:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:05.074+00:00 2026-10-15T05:05:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:06.079+00:00 2026-10-15T05:05:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:07.085+00:00 2026-10-15T05:05:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:08.092+00:00 2026-10-15T05:05:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:09.097+00:00 2026-10-15T05:05:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:10.100+00:00 2026-10-15T05:05:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:11.106+00:00 2026-10-15T05:05:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:12.112+00:00 2026-10-15T05:05:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:13.118+00:00 2026-10-15T05:05:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:14.123+00:00 2026-10-15T05:05:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:15.129+00:00 2026-10-15T05:05:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:16.134+00:00 2026-10-15T05:05:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:17.143+00:00 2026-10-15T05:05:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:18.143+00:00 2026-10-15T05:05:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:19.148+00:00 2026-10-15T05:05:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:20.152+00:00 2026-10-15T05:05:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:21.158+00:00 2026-10-15T05:05:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:22.163+00:00 2026-10-15T05:05:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:23.168+00:00 2026-10-15T05:05:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:24.179+00:00 2026-10-15T05:05:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:25.179+00:00 2026-10-15T05:05:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:26.184+00:00 2026-10-15T05:05:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:27.195+00:00 2026-10-15T05:05:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:28.194+00:00 2026-10-15T05:05:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:29.198+00:00 2026-10-15T05:05:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:30.204+00:00 2026-10-15T05:05:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:31.210+00:00 2026-10-15T05:05:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:32.214+00:00 2026-10-15T05:05:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:33.220+00:00 2026-10-15T05:05:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:34.225+00:00 2026-10-15T05:05:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:35.231+00:00 2026-10-15T05:05:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:36.236+00:00 2026-10-15T05:05:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:37.242+00:00 2026-10-15T05:05:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:38.247+00:00 2026-10-15T05:05:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:39.252+00:00 2026-10-15T05:05:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:40.257+00:00 2026-10-15T05:05:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:41.262+00:00 2026-10-15T05:05:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:42.267+00:00 2026-10-15T05:05:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:43.272+00:00 2026-10-15T05:05:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:44.278+00:00 2026-10-15T05:05:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:45.282+00:00 2026-10-15T05:05:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:46.287+00:00 2026-10-15T05:05:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:47.292+00:00 2026-10-15T05:05:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:48.365+00:00 2026-10-15T05:05:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:49.304+00:00 2026-10-15T05:05:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:50.306+00:00 2026-10-15T05:05:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:51.313+00:00 2026-10-15T05:05:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:52.317+00:00 2026-10-15T05:05:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:53.322+00:00 2026-10-15T05:05:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:54.328+00:00 2026-10-15T05:05:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:55.344+00:00 2026-10-15T05:05:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:56.338+00:00 2026-10-15T05:05:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:57.347+00:00 2026-10-15T05:05:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:58.348+00:00 2026-10-15T05:05:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:05:59.353+00:00 2026-10-15T05:05:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:00.358+00:00 2026-10-15T05:06:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:01.365+00:00 2026-10-15T05:06:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:02.370+00:00 2026-10-15T05:06:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:03.375+00:00 2026-10-15T05:06:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:04.381+00:00 2026-10-15T05:06:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:05.388+00:00 2026-10-15T05:06:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:06.400+00:00 2026-10-15T05:06:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:07.395+00:00 2026-10-15T05:06:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:08.401+00:00 2026-10-15T05:06:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:09.406+00:00 2026-10-15T05:06:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:10.411+00:00 2026-10-15T05:06:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:11.416+00:00 2026-10-15T05:06:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:12.422+00:00 2026-10-15T05:06:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:13.427+00:00 2026-10-15T05:06:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:14.433+00:00 2026-10-15T05:06:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:15.437+00:00 2026-10-15T05:06:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:16.457+00:00 2026-10-15T05:06:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:17.453+00:00 2026-10-15T05:06:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:18.453+00:00 2026-10-15T05:06:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:19.458+00:00 2026-10-15T05:06:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:20.463+00:00 2026-10-15T05:06:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:21.468+00:00 2026-10-15T05:06:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:22.472+00:00 2026-10-15T05:06:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:23.478+00:00 2026-10-15T05:06:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:24.483+00:00 2026-10-15T05:06:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:25.488+00:00 2026-10-15T05:06:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:26.493+00:00 2026-10-15T05:06:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:27.498+00:00 2026-10-15T05:06:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:28.517+00:00 2026-10-15T05:06:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:29.508+00:00 2026-10-15T05:06:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:30.513+00:00 2026-10-15T05:06:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:31.527+00:00 2026-10-15T05:06:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:32.523+00:00 2026-10-15T05:06:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:33.528+00:00 2026-10-15T05:06:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:34.533+00:00 2026-10-15T05:06:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:35.607+00:00 2026-10-15T05:06:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:36.545+00:00 2026-10-15T05:06:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:37.548+00:00 2026-10-15T05:06:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:38.553+00:00 2026-10-15T05:06:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:39.558+00:00 2026-10-15T05:06:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:40.563+00:00 2026-10-15T05:06:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:41.569+00:00 2026-10-15T05:06:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:42.574+00:00 2026-10-15T05:06:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:43.579+00:00 2026-10-15T05:06:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:44.591+00:00 2026-10-15T05:06:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:45.589+00:00 2026-10-15T05:06:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:46.600+00:00 2026-10-15T05:06:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:47.598+00:00 2026-10-15T05:06:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:47.598+00:00 2026-10-15T05:06:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:48.603+00:00 2026-10-15T05:06:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:48.603+00:00 2026-10-15T05:06:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:49.608+00:00 2026-10-15T05:06:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:49.608+00:00 2026-10-15T05:06:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:50.613+00:00 2026-10-15T05:06:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:50.613+00:00 2026-10-15T05:06:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:51.618+00:00 2026-10-15T05:06:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:51.618+00:00 2026-10-15T05:06:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:52.623+00:00 2026-10-15T05:06:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:52.623+00:00 2026-10-15T05:06:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:53.635+00:00 2026-10-15T05:06:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:53.635+00:00 2026-10-15T05:06:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:54.632+00:00 2026-10-15T05:06:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:54.632+00:00 2026-10-15T05:06:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:55.637+00:00 2026-10-15T05:06:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:55.637+00:00 2026-10-15T05:06:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:56.775+00:00 2026-10-15T05:06:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:56.775+00:00 2026-10-15T05:06:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:57.648+00:00 2026-10-15T05:06:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:57.648+00:00 2026-10-15T05:06:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:58.652+00:00 2026-10-15T05:06:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:58.652+00:00 2026-10-15T05:06:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:06:59.709+00:00 2026-10-15T05:06:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:06:59.709+00:00 2026-10-15T05:06:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:00.664+00:00 2026-10-15T05:07:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:00.664+00:00 2026-10-15T05:07:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:01.667+00:00 2026-10-15T05:07:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:01.667+00:00 2026-10-15T05:07:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:02.672+00:00 2026-10-15T05:07:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:02.672+00:00 2026-10-15T05:07:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:03.749+00:00 2026-10-15T05:07:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:03.749+00:00 2026-10-15T05:07:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:04.681+00:00 2026-10-15T05:07:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:04.681+00:00 2026-10-15T05:07:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:05.687+00:00 2026-10-15T05:07:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:05.687+00:00 2026-10-15T05:07:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:06.692+00:00 2026-10-15T05:07:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:06.692+00:00 2026-10-15T05:07:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:07.700+00:00 2026-10-15T05:07:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:07.700+00:00 2026-10-15T05:07:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:08.701+00:00 2026-10-15T05:07:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:08.701+00:00 2026-10-15T05:07:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:09.705+00:00 2026-10-15T05:07:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:09.705+00:00 2026-10-15T05:07:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:10.714+00:00 2026-10-15T05:07:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:10.714+00:00 2026-10-15T05:07:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:11.716+00:00 2026-10-15T05:07:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:11.716+00:00 2026-10-15T05:07:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:12.720+00:00 2026-10-15T05:07:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:12.720+00:00 2026-10-15T05:07:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:13.725+00:00 2026-10-15T05:07:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:13.725+00:00 2026-10-15T05:07:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:14.730+00:00 2026-10-15T05:07:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:14.731+00:00 2026-10-15T05:07:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:15.735+00:00 2026-10-15T05:07:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:15.735+00:00 2026-10-15T05:07:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:16.740+00:00 2026-10-15T05:07:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:16.740+00:00 2026-10-15T05:07:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:17.746+00:00 2026-10-15T05:07:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:17.746+00:00 2026-10-15T05:07:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:18.750+00:00 2026-10-15T05:07:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:18.750+00:00 2026-10-15T05:07:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:19.756+00:00 2026-10-15T05:07:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:19.757+00:00 2026-10-15T05:07:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:20.760+00:00 2026-10-15T05:07:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:20.761+00:00 2026-10-15T05:07:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:21.765+00:00 2026-10-15T05:07:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:21.765+00:00 2026-10-15T05:07:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:22.770+00:00 2026-10-15T05:07:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:22.770+00:00 2026-10-15T05:07:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:23.775+00:00 2026-10-15T05:07:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:23.775+00:00 2026-10-15T05:07:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:24.781+00:00 2026-10-15T05:07:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:24.781+00:00 2026-10-15T05:07:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:25.786+00:00 2026-10-15T05:07:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:25.786+00:00 2026-10-15T05:07:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:26.790+00:00 2026-10-15T05:07:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:26.790+00:00 2026-10-15T05:07:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:27.902+00:00 2026-10-15T05:07:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:27.902+00:00 2026-10-15T05:07:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:28.800+00:00 2026-10-15T05:07:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:28.800+00:00 2026-10-15T05:07:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:29.805+00:00 2026-10-15T05:07:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:29.805+00:00 2026-10-15T05:07:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:30.812+00:00 2026-10-15T05:07:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:30.812+00:00 2026-10-15T05:07:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:31.816+00:00 2026-10-15T05:07:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:31.816+00:00 2026-10-15T05:07:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:32.821+00:00 2026-10-15T05:07:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:32.821+00:00 2026-10-15T05:07:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:33.826+00:00 2026-10-15T05:07:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:33.826+00:00 2026-10-15T05:07:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:34.832+00:00 2026-10-15T05:07:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:34.833+00:00 2026-10-15T05:07:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:35.837+00:00 2026-10-15T05:07:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:35.837+00:00 2026-10-15T05:07:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:36.843+00:00 2026-10-15T05:07:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:36.843+00:00 2026-10-15T05:07:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:37.847+00:00 2026-10-15T05:07:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:37.847+00:00 2026-10-15T05:07:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:38.852+00:00 2026-10-15T05:07:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:38.853+00:00 2026-10-15T05:07:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:39.859+00:00 2026-10-15T05:07:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:39.859+00:00 2026-10-15T05:07:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:40.864+00:00 2026-10-15T05:07:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:40.864+00:00 2026-10-15T05:07:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:41.869+00:00 2026-10-15T05:07:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:41.869+00:00 2026-10-15T05:07:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:42.875+00:00 2026-10-15T05:07:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:42.875+00:00 2026-10-15T05:07:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:43.880+00:00 2026-10-15T05:07:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:43.880+00:00 2026-10-15T05:07:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:44.890+00:00 2026-10-15T05:07:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:44.890+00:00 2026-10-15T05:07:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:45.890+00:00 2026-10-15T05:07:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:45.890+00:00 2026-10-15T05:07:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:46.904+00:00 2026-10-15T05:07:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:46.905+00:00 2026-10-15T05:07:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:47.899+00:00 2026-10-15T05:07:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:47.899+00:00 2026-10-15T05:07:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:48.906+00:00 2026-10-15T05:07:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:48.906+00:00 2026-10-15T05:07:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:49.910+00:00 2026-10-15T05:07:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:49.910+00:00 2026-10-15T05:07:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:50.916+00:00 2026-10-15T05:07:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:50.916+00:00 2026-10-15T05:07:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:51.921+00:00 2026-10-15T05:07:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:51.922+00:00 2026-10-15T05:07:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:52.928+00:00 2026-10-15T05:07:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:52.929+00:00 2026-10-15T05:07:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:53.931+00:00 2026-10-15T05:07:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:53.931+00:00 2026-10-15T05:07:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:54.935+00:00 2026-10-15T05:07:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:54.935+00:00 2026-10-15T05:07:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:55.951+00:00 2026-10-15T05:07:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:55.951+00:00 2026-10-15T05:07:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:56.945+00:00 2026-10-15T05:07:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:56.945+00:00 2026-10-15T05:07:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:57.974+00:00 2026-10-15T05:07:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:57.974+00:00 2026-10-15T05:07:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:58.955+00:00 2026-10-15T05:07:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:58.955+00:00 2026-10-15T05:07:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:07:59.960+00:00 2026-10-15T05:07:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:07:59.960+00:00 2026-10-15T05:07:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:00.965+00:00 2026-10-15T05:08:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:00.965+00:00 2026-10-15T05:08:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:01.970+00:00 2026-10-15T05:08:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:01.970+00:00 2026-10-15T05:08:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:02.978+00:00 2026-10-15T05:08:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:02.978+00:00 2026-10-15T05:08:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:03.984+00:00 2026-10-15T05:08:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:03.984+00:00 2026-10-15T05:08:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:04.990+00:00 2026-10-15T05:08:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:04.990+00:00 2026-10-15T05:08:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:06.000+00:00 2026-10-15T05:08:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:06.000+00:00 2026-10-15T05:08:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:07.000+00:00 2026-10-15T05:08:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:07.000+00:00 2026-10-15T05:08:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:08.008+00:00 2026-10-15T05:08:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:08.008+00:00 2026-10-15T05:08:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:09.010+00:00 2026-10-15T05:08:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:09.010+00:00 2026-10-15T05:08:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:10.016+00:00 2026-10-15T05:08:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:10.016+00:00 2026-10-15T05:08:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:11.019+00:00 2026-10-15T05:08:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:11.020+00:00 2026-10-15T05:08:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:12.025+00:00 2026-10-15T05:08:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:12.026+00:00 2026-10-15T05:08:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:13.030+00:00 2026-10-15T05:08:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:13.030+00:00 2026-10-15T05:08:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:14.048+00:00 2026-10-15T05:08:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:14.048+00:00 2026-10-15T05:08:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:15.040+00:00 2026-10-15T05:08:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:15.040+00:00 2026-10-15T05:08:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:16.049+00:00 2026-10-15T05:08:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:16.050+00:00 2026-10-15T05:08:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:17.053+00:00 2026-10-15T05:08:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:17.053+00:00 2026-10-15T05:08:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:18.058+00:00 2026-10-15T05:08:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:18.058+00:00 2026-10-15T05:08:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:19.062+00:00 2026-10-15T05:08:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:19.062+00:00 2026-10-15T05:08:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:20.066+00:00 2026-10-15T05:08:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:20.066+00:00 2026-10-15T05:08:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:21.076+00:00 2026-10-15T05:08:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:21.076+00:00 2026-10-15T05:08:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:22.078+00:00 2026-10-15T05:08:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:22.078+00:00 2026-10-15T05:08:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:23.084+00:00 2026-10-15T05:08:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:23.084+00:00 2026-10-15T05:08:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:24.092+00:00 2026-10-15T05:08:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:24.092+00:00 2026-10-15T05:08:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:25.094+00:00 2026-10-15T05:08:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:25.094+00:00 2026-10-15T05:08:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:26.100+00:00 2026-10-15T05:08:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:26.100+00:00 2026-10-15T05:08:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:27.103+00:00 2026-10-15T05:08:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:27.104+00:00 2026-10-15T05:08:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:28.108+00:00 2026-10-15T05:08:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:28.108+00:00 2026-10-15T05:08:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:29.113+00:00 2026-10-15T05:08:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:29.113+00:00 2026-10-15T05:08:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:30.119+00:00 2026-10-15T05:08:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:30.119+00:00 2026-10-15T05:08:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:31.124+00:00 2026-10-15T05:08:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:31.124+00:00 2026-10-15T05:08:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:32.129+00:00 2026-10-15T05:08:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:32.129+00:00 2026-10-15T05:08:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:33.133+00:00 2026-10-15T05:08:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:33.133+00:00 2026-10-15T05:08:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:34.139+00:00 2026-10-15T05:08:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:34.139+00:00 2026-10-15T05:08:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:35.143+00:00 2026-10-15T05:08:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:35.143+00:00 2026-10-15T05:08:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:36.321+00:00 2026-10-15T05:08:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:36.321+00:00 2026-10-15T05:08:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:37.153+00:00 2026-10-15T05:08:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:37.153+00:00 2026-10-15T05:08:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:38.158+00:00 2026-10-15T05:08:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:38.158+00:00 2026-10-15T05:08:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:39.178+00:00 2026-10-15T05:08:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:39.178+00:00 2026-10-15T05:08:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:40.168+00:00 2026-10-15T05:08:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:40.168+00:00 2026-10-15T05:08:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:41.172+00:00 2026-10-15T05:08:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:41.172+00:00 2026-10-15T05:08:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:42.179+00:00 2026-10-15T05:08:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:42.179+00:00 2026-10-15T05:08:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:43.183+00:00 2026-10-15T05:08:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:43.183+00:00 2026-10-15T05:08:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:44.188+00:00 2026-10-15T05:08:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:44.188+00:00 2026-10-15T05:08:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:45.192+00:00 2026-10-15T05:08:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:45.193+00:00 2026-10-15T05:08:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:46.198+00:00 2026-10-15T05:08:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:46.198+00:00 2026-10-15T05:08:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:47.203+00:00 2026-10-15T05:08:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:47.203+00:00 2026-10-15T05:08:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:48.207+00:00 2026-10-15T05:08:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:48.207+00:00 2026-10-15T05:08:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:49.212+00:00 2026-10-15T05:08:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:49.212+00:00 2026-10-15T05:08:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:50.217+00:00 2026-10-15T05:08:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:50.217+00:00 2026-10-15T05:08:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:51.256+00:00 2026-10-15T05:08:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:51.257+00:00 2026-10-15T05:08:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:52.226+00:00 2026-10-15T05:08:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:52.226+00:00 2026-10-15T05:08:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:53.231+00:00 2026-10-15T05:08:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:53.231+00:00 2026-10-15T05:08:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:54.236+00:00 2026-10-15T05:08:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:54.236+00:00 2026-10-15T05:08:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:55.241+00:00 2026-10-15T05:08:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:55.241+00:00 2026-10-15T05:08:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:56.247+00:00 2026-10-15T05:08:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:56.247+00:00 2026-10-15T05:08:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:57.269+00:00 2026-10-15T05:08:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:57.269+00:00 2026-10-15T05:08:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:58.258+00:00 2026-10-15T05:08:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:58.258+00:00 2026-10-15T05:08:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:08:59.262+00:00 2026-10-15T05:08:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:08:59.262+00:00 2026-10-15T05:08:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:00.271+00:00 2026-10-15T05:09:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:00.271+00:00 2026-10-15T05:09:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:01.272+00:00 2026-10-15T05:09:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:01.272+00:00 2026-10-15T05:09:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:02.283+00:00 2026-10-15T05:09:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:02.283+00:00 2026-10-15T05:09:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:03.282+00:00 2026-10-15T05:09:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:03.282+00:00 2026-10-15T05:09:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:04.291+00:00 2026-10-15T05:09:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:04.291+00:00 2026-10-15T05:09:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:05.293+00:00 2026-10-15T05:09:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:05.294+00:00 2026-10-15T05:09:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:06.299+00:00 2026-10-15T05:09:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:06.299+00:00 2026-10-15T05:09:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:07.304+00:00 2026-10-15T05:09:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:07.304+00:00 2026-10-15T05:09:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:08.307+00:00 2026-10-15T05:09:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:08.308+00:00 2026-10-15T05:09:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:09.312+00:00 2026-10-15T05:09:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:09.312+00:00 2026-10-15T05:09:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:10.318+00:00 2026-10-15T05:09:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:10.318+00:00 2026-10-15T05:09:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:11.321+00:00 2026-10-15T05:09:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:11.321+00:00 2026-10-15T05:09:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:12.327+00:00 2026-10-15T05:09:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:12.328+00:00 2026-10-15T05:09:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:13.332+00:00 2026-10-15T05:09:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:13.333+00:00 2026-10-15T05:09:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:14.392+00:00 2026-10-15T05:09:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:14.392+00:00 2026-10-15T05:09:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:15.342+00:00 2026-10-15T05:09:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:15.342+00:00 2026-10-15T05:09:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:16.348+00:00 2026-10-15T05:09:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:16.348+00:00 2026-10-15T05:09:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:17.352+00:00 2026-10-15T05:09:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:17.353+00:00 2026-10-15T05:09:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:18.356+00:00 2026-10-15T05:09:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:18.356+00:00 2026-10-15T05:09:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:19.362+00:00 2026-10-15T05:09:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:19.362+00:00 2026-10-15T05:09:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:20.370+00:00 2026-10-15T05:09:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:20.370+00:00 2026-10-15T05:09:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:21.372+00:00 2026-10-15T05:09:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:21.372+00:00 2026-10-15T05:09:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:22.392+00:00 2026-10-15T05:09:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:22.392+00:00 2026-10-15T05:09:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:23.381+00:00 2026-10-15T05:09:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:23.381+00:00 2026-10-15T05:09:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:24.386+00:00 2026-10-15T05:09:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:24.386+00:00 2026-10-15T05:09:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:25.391+00:00 2026-10-15T05:09:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:25.391+00:00 2026-10-15T05:09:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:26.397+00:00 2026-10-15T05:09:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:26.397+00:00 2026-10-15T05:09:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:27.404+00:00 2026-10-15T05:09:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:27.404+00:00 2026-10-15T05:09:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:28.411+00:00 2026-10-15T05:09:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:28.412+00:00 2026-10-15T05:09:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:29.416+00:00 2026-10-15T05:09:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:29.416+00:00 2026-10-15T05:09:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:30.417+00:00 2026-10-15T05:09:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:30.417+00:00 2026-10-15T05:09:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:31.515+00:00 2026-10-15T05:09:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:31.515+00:00 2026-10-15T05:09:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:32.426+00:00 2026-10-15T05:09:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:32.426+00:00 2026-10-15T05:09:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:33.431+00:00 2026-10-15T05:09:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:33.431+00:00 2026-10-15T05:09:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:34.436+00:00 2026-10-15T05:09:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:34.437+00:00 2026-10-15T05:09:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:35.443+00:00 2026-10-15T05:09:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:35.443+00:00 2026-10-15T05:09:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:36.447+00:00 2026-10-15T05:09:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:36.447+00:00 2026-10-15T05:09:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:37.451+00:00 2026-10-15T05:09:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:37.451+00:00 2026-10-15T05:09:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:38.460+00:00 2026-10-15T05:09:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:38.460+00:00 2026-10-15T05:09:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:39.462+00:00 2026-10-15T05:09:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:39.462+00:00 2026-10-15T05:09:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:40.467+00:00 2026-10-15T05:09:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:40.467+00:00 2026-10-15T05:09:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:41.472+00:00 2026-10-15T05:09:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:41.472+00:00 2026-10-15T05:09:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:42.477+00:00 2026-10-15T05:09:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:42.477+00:00 2026-10-15T05:09:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:43.483+00:00 2026-10-15T05:09:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:43.483+00:00 2026-10-15T05:09:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:44.489+00:00 2026-10-15T05:09:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:44.489+00:00 2026-10-15T05:09:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:45.496+00:00 2026-10-15T05:09:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:45.496+00:00 2026-10-15T05:09:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:46.499+00:00 2026-10-15T05:09:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:46.499+00:00 2026-10-15T05:09:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:47.504+00:00 2026-10-15T05:09:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:47.504+00:00 2026-10-15T05:09:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:48.509+00:00 2026-10-15T05:09:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:48.509+00:00 2026-10-15T05:09:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:49.515+00:00 2026-10-15T05:09:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:49.515+00:00 2026-10-15T05:09:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:50.520+00:00 2026-10-15T05:09:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:50.521+00:00 2026-10-15T05:09:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:51.525+00:00 2026-10-15T05:09:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:51.526+00:00 2026-10-15T05:09:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:52.531+00:00 2026-10-15T05:09:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:52.531+00:00 2026-10-15T05:09:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:53.542+00:00 2026-10-15T05:09:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:53.542+00:00 2026-10-15T05:09:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:54.542+00:00 2026-10-15T05:09:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:54.542+00:00 2026-10-15T05:09:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:55.546+00:00 2026-10-15T05:09:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:55.546+00:00 2026-10-15T05:09:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:56.552+00:00 2026-10-15T05:09:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:56.552+00:00 2026-10-15T05:09:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:57.558+00:00 2026-10-15T05:09:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:57.559+00:00 2026-10-15T05:09:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:58.573+00:00 2026-10-15T05:09:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:58.573+00:00 2026-10-15T05:09:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:09:59.566+00:00 2026-10-15T05:09:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:09:59.566+00:00 2026-10-15T05:09:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:00.573+00:00 2026-10-15T05:10:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:00.573+00:00 2026-10-15T05:10:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:01.579+00:00 2026-10-15T05:10:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:01.579+00:00 2026-10-15T05:10:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:02.586+00:00 2026-10-15T05:10:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:02.586+00:00 2026-10-15T05:10:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:03.595+00:00 2026-10-15T05:10:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:03.596+00:00 2026-10-15T05:10:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:04.597+00:00 2026-10-15T05:10:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:04.597+00:00 2026-10-15T05:10:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:05.603+00:00 2026-10-15T05:10:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:05.603+00:00 2026-10-15T05:10:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:06.609+00:00 2026-10-15T05:10:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:06.609+00:00 2026-10-15T05:10:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:07.618+00:00 2026-10-15T05:10:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:07.618+00:00 2026-10-15T05:10:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:08.623+00:00 2026-10-15T05:10:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:08.623+00:00 2026-10-15T05:10:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:09.629+00:00 2026-10-15T05:10:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:09.629+00:00 2026-10-15T05:10:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:10.636+00:00 2026-10-15T05:10:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:10.636+00:00 2026-10-15T05:10:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:11.642+00:00 2026-10-15T05:10:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:11.642+00:00 2026-10-15T05:10:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:12.647+00:00 2026-10-15T05:10:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:12.647+00:00 2026-10-15T05:10:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:13.651+00:00 2026-10-15T05:10:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:13.651+00:00 2026-10-15T05:10:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:14.657+00:00 2026-10-15T05:10:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:14.658+00:00 2026-10-15T05:10:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:15.803+00:00 2026-10-15T05:10:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:15.803+00:00 2026-10-15T05:10:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:16.680+00:00 2026-10-15T05:10:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:16.680+00:00 2026-10-15T05:10:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:17.672+00:00 2026-10-15T05:10:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:17.672+00:00 2026-10-15T05:10:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:18.710+00:00 2026-10-15T05:10:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:18.711+00:00 2026-10-15T05:10:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:19.683+00:00 2026-10-15T05:10:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:19.683+00:00 2026-10-15T05:10:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:20.688+00:00 2026-10-15T05:10:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:20.688+00:00 2026-10-15T05:10:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:21.693+00:00 2026-10-15T05:10:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:21.694+00:00 2026-10-15T05:10:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:22.699+00:00 2026-10-15T05:10:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:22.699+00:00 2026-10-15T05:10:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:23.703+00:00 2026-10-15T05:10:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:23.703+00:00 2026-10-15T05:10:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:24.708+00:00 2026-10-15T05:10:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:24.708+00:00 2026-10-15T05:10:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:25.714+00:00 2026-10-15T05:10:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:25.715+00:00 2026-10-15T05:10:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:26.720+00:00 2026-10-15T05:10:26Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:26.720+00:00 2026-10-15T05:10:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:26.720+00:00 2026-10-15T05:10:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:27.726+00:00 2026-10-15T05:10:27Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:27.726+00:00 2026-10-15T05:10:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:27.726+00:00 2026-10-15T05:10:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:28.730+00:00 2026-10-15T05:10:28Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:28.731+00:00 2026-10-15T05:10:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:28.731+00:00 2026-10-15T05:10:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:29.741+00:00 2026-10-15T05:10:29Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:29.741+00:00 2026-10-15T05:10:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:29.741+00:00 2026-10-15T05:10:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:30.740+00:00 2026-10-15T05:10:30Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:30.740+00:00 2026-10-15T05:10:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:30.740+00:00 2026-10-15T05:10:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:31.746+00:00 2026-10-15T05:10:31Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:31.746+00:00 2026-10-15T05:10:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:31.746+00:00 2026-10-15T05:10:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:32.755+00:00 2026-10-15T05:10:32Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:32.755+00:00 2026-10-15T05:10:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:32.755+00:00 2026-10-15T05:10:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:33.755+00:00 2026-10-15T05:10:33Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:33.756+00:00 2026-10-15T05:10:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:33.756+00:00 2026-10-15T05:10:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:10:34.761+00:00 2026-10-15T05:10:34Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:10:34.761+00:00 2026-10-15T05:10:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:10:34.761+00:00 2026-10-15T05:10:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log deleted file mode 100644 index 3f20cd2a..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T05:04:59.071+00:00 2026-10-15T05:04:59.090098465Z {"level":"info","ts":"2026-10-15T05:04:59Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"065c25b1-0fba-4abe-8200-01346b4d1314","claim":"claim-lora-qwen-qwen3-8b-0-trainer-125fac70","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} -2026-10-15T05:06:58.207+00:00 2026-10-15T05:06:58.027203485Z {"level":"info","ts":"2026-10-15T05:06:58Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"ccc8816d-971a-4ae8-bad0-64691bfb74c6","claim":"claim-lora-qwen-qwen3-8b-0-sampler-3920e58e","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json deleted file mode 100644 index c2297ac3..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/timeline.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "case": "after-C1-8b-h100-cold", - "collected": "2026-10-15T05:10:35.141+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 05:04:58.138000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen3-8B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 05:04:58.139000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 05:06:49.594000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "048101ca-2c60-4c63-afa2-2ae75e83f1c4" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 05:06:52.784000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 05:06:55.595000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 05:06:57.617000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 05:06:57.618000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 05:06:59.533000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 05:06:59.534000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 05:10:27.431000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4\n\nQuestion: What is " - }, - "client.end": { - "ts": "2026-10-15 05:10:27.434000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen3-8b-0-sampler", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "pod": "orw-lora-qwen-qwen3-8b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 05:06:57+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:06:56.988000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 05:06:56.988000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 05:06:56.988000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 05:08:28+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 05:06:58+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 05:06:58+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 05:08:28+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 05:08:28+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 05:06:58+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 05:08:28+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 05:06:58+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "sampler.event_Pulling": { - "ts": "2026-10-15 05:07:01+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 05:08:28+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.45", - "kubelet_pull_s": 86.459, - "pulled_from_cache": false - }, - "sampler.event_Created": { - "ts": "2026-10-15 05:08:28+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 05:08:28+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 05:08:35.508944+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 05:08:52.969476+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:08:52 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 05:08:54.079567+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:08:54 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 05:08:55.068664+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:08:55 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 05:09:08.096254+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:09:08 [default_loader.py:430] Loading weights took 13.03 seconds", - "self_reported_s": 13.03 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 05:09:08.762926+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:09:08 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.956827 seconds", - "self_reported_s": 14.956827 - }, - "sampler.compile_done": { - "ts": "2026-10-15 05:09:45.574813+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:09:45 [monitor.py:53] torch.compile took 35.82 s in total", - "self_reported_s": 35.82 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 05:10:06.072864+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:10:06 [model_runner.py:1066] Graph capturing finished in 17 secs, took 0.78 GiB", - "self_reported_s": 17.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 05:10:06.701571+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:10:06 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 05:10:25.431150+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:10:25 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.66 s", - "self_reported_s": 76.66 - }, - "sampler.ready_key": { - "ts": "2026-10-15 05:10:26+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - }, - "trainer": { - "workload": "lora-qwen-qwen3-8b-0-trainer", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "pod": "orw-lora-qwen-qwen3-8b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 05:04:58+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:04:58.742000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 05:04:58.742000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 05:05:00.362000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 05:06:30+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 05:04:59+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 05:04:59+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 05:06:29+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 05:06:29+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 05:04:59+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 05:06:29+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 05:04:59+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "trainer.event_Pulling": { - "ts": "2026-10-15 05:05:02+00:00", - "source": "event: Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 05:06:29+00:00", - "source": "event: Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.47", - "kubelet_pull_s": 86.476, - "pulled_from_cache": false - }, - "trainer.event_Created": { - "ts": "2026-10-15 05:06:29+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 05:06:29+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 05:06:33.850821+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 05:06:34.074918+00:00", - "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 05:06:47.108478+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 05:06:47.110263+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 05:06:47+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt deleted file mode 100644 index 9c360d9c..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:04:53.958+00:00 diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt deleted file mode 100644 index a9917932..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json b/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json deleted file mode 100644 index 22c659c3..00000000 --- a/runs/2026-10-15_cold_start/after-C1-8b-h100-cold/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:57Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040908682239008", - "uid": "3920e58e-f415-4c96-8485-77a8cd538872" - }, - "spec": { - "accelerator": { - "memory": "29Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "44791173-6b7c-4d22-9a8b-46ef83190384", - "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:08:28Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-3920e58e", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "29Gi", - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:58Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040790072463004", - "uid": "125fac70-8359-4f66-bcb5-20df4c1e1764" - }, - "spec": { - "accelerator": { - "memory": "20Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "371ea4b9-ae90-4e15-a7b1-b34802637e23", - "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-125fac70", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:06:30Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-125fac70", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "20Gi", - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log deleted file mode 100644 index 2b3010d6..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached.run.log +++ /dev/null @@ -1,35 +0,0 @@ -watching; recorders [18410, 18411, 18413, 18415, 18417, 18412, 18414, 18416] write into runs/2026-10-15_cold_start/after-D1-8b-h100-cached -collected 2 worker log(s) into runs/2026-10-15_cold_start/after-D1-8b-h100-cached -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.8 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 3.0 | -| container create | trainer.event_Pulled | trainer.container_started | 0.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.5 | -| weights load | trainer.load_start | trainer.load_done | 13.5 | -| ready key | trainer.load_done | trainer.ready_key | -27.0 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 25.3 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | -0.4 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | -| container create | sampler.event_Pulled | sampler.container_started | 0.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 15.4 | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 9.5 | -| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 12.9 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.6 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.3 | -| ready key | sampler.engine_init_done | sampler.ready_key | 0.2 | -| client: first sample blocked | client.sample.start | client.sample.done | 119.4 | - diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log deleted file mode 100644 index 43b56b1e..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/api-server.log +++ /dev/null @@ -1,32 +0,0 @@ -2026-10-15T05:11:45.416+00:00 2026-10-15T05:11:45.437391305Z INFO: 127.0.0.1:34326 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T05:11:45.652+00:00 2026-10-15T05:11:45.675793966Z INFO: 127.0.0.1:34340 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T05:11:45.721+00:00 2026-10-15T05:11:45.743584668Z INFO: 127.0.0.1:34340 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T05:11:45.982+00:00 2026-10-15T05:11:46.006037569Z [API_SERVER] enqueued op=create_model request_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:11:45.984+00:00 2026-10-15T05:11:46.008026346Z INFO: 127.0.0.1:34350 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T05:11:50.728+00:00 2026-10-15T05:11:50.746569548Z INFO: 10.32.3.1:39220 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:11:55.956+00:00 2026-10-15T05:11:55.977933873Z INFO: 127.0.0.1:58848 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:12:00.725+00:00 2026-10-15T05:12:00.746355493Z INFO: 10.32.3.1:44030 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:12:10.663+00:00 2026-10-15T05:12:10.683497651Z INFO: 127.0.0.1:51194 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T05:12:10.723+00:00 2026-10-15T05:12:10.747036803Z INFO: 10.32.3.1:38468 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:12:13.399+00:00 2026-10-15T05:12:13.422272335Z [API_SERVER] enqueued op=forward_backward request_id=641d3bab-a7c0-4047-b609-7ca88e07767a model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:12:13.399+00:00 2026-10-15T05:12:13.423605880Z INFO: 127.0.0.1:51194 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T05:12:16.334+00:00 2026-10-15T05:12:16.355910312Z [API_SERVER] enqueued op=optim_step request_id=cb0f1a0a-f9af-4b9d-9aeb-e82e91a14640 model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:12:16.335+00:00 2026-10-15T05:12:16.356438363Z INFO: 127.0.0.1:51194 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T05:12:18.459+00:00 2026-10-15T05:12:18.478776460Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=bfeec1f4-669e-4b5b-b8c0-b876b9cae9dc model_id=428ca04c-2ae6-41ed-9a4b-94f1cd79c97e active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:12:18.459+00:00 2026-10-15T05:12:18.479377759Z INFO: 127.0.0.1:51194 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T05:12:20.456+00:00 2026-10-15T05:12:20.479018182Z INFO: 127.0.0.1:35222 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T05:12:20.723+00:00 2026-10-15T05:12:20.747005620Z INFO: 10.32.3.1:38880 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:12:30.725+00:00 2026-10-15T05:12:30.747249473Z INFO: 10.32.3.1:33044 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:12:40.724+00:00 2026-10-15T05:12:40.746273596Z INFO: 10.32.3.1:41864 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:12:50.725+00:00 2026-10-15T05:12:50.746808349Z INFO: 10.32.3.1:41618 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:13:00.731+00:00 2026-10-15T05:13:00.748054823Z INFO: 10.32.3.1:46394 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:13:06.154+00:00 2026-10-15T05:13:06.171189246Z INFO: 127.0.0.1:44254 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:13:10.724+00:00 2026-10-15T05:13:10.747443779Z INFO: 10.32.3.1:38328 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:13:20.726+00:00 2026-10-15T05:13:20.747144519Z INFO: 10.32.3.1:44212 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:13:30.727+00:00 2026-10-15T05:13:30.746988779Z INFO: 10.32.3.1:39326 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:13:40.725+00:00 2026-10-15T05:13:40.747152571Z INFO: 10.32.3.1:53070 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:13:50.726+00:00 2026-10-15T05:13:50.746993399Z INFO: 10.32.3.1:51424 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:13:56.352+00:00 2026-10-15T05:13:56.369899995Z INFO: 127.0.0.1:35362 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:14:00.724+00:00 2026-10-15T05:14:00.747118474Z INFO: 10.32.3.1:46958 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:14:10.725+00:00 2026-10-15T05:14:10.747141725Z INFO: 10.32.3.1:35928 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:14:20.724+00:00 2026-10-15T05:14:20.746773992Z INFO: 10.32.3.1:39268 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl deleted file mode 100644 index a1728906..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T05:11:45.156+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T05:11:45.157+00:00", "t": 0.001, "phase": "create_model.start"} -{"ts": "2026-10-15T05:12:10.447+00:00", "t": 25.291, "phase": "create_model.done", "model_id": "428ca04c-2ae6-41ed-9a4b-94f1cd79c97e"} -{"ts": "2026-10-15T05:12:13.323+00:00", "t": 28.167, "phase": "forward_backward.start"} -{"ts": "2026-10-15T05:12:16.232+00:00", "t": 31.076, "phase": "forward_backward.done"} -{"ts": "2026-10-15T05:12:18.352+00:00", "t": 33.196, "phase": "optim_step.done"} -{"ts": "2026-10-15T05:12:18.353+00:00", "t": 33.197, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T05:12:20.273+00:00", "t": 35.117, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T05:12:20.275+00:00", "t": 35.118, "phase": "sample.start"} -{"ts": "2026-10-15T05:14:19.688+00:00", "t": 154.532, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} -{"ts": "2026-10-15T05:14:19.689+00:00", "t": 154.533, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log deleted file mode 100644 index 832f9dbe..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.001s create_model.start -[cold-start] 25.291s create_model.done {'model_id': '428ca04c-2ae6-41ed-9a4b-94f1cd79c97e'} -[cold-start] 28.167s forward_backward.start -[cold-start] 31.076s forward_backward.done -[cold-start] 33.196s optim_step.done -[cold-start] 33.197s save_weights_and_get_sampling_client.start -[cold-start] 35.117s save_weights_and_get_sampling_client.done -[cold-start] 35.118s sample.start -[cold-start] 154.532s sample.done {'text': ' 4\n\nQuestion: What is '} -[cold-start] 154.533s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt deleted file mode 100644 index dabd9d84..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:14:20.211+00:00 diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json deleted file mode 100644 index 4e274608..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/events.json +++ /dev/null @@ -1,2743 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:10Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:10Z", - "name": "cold-start-fio.18de9a0ef295773d", - "namespace": "openrl-system", - "resourceVersion": "1792039090760383011", - "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5be63862", - "namespace": "openrl-system", - "resourceVersion": "1792039092515007000", - "uid": "355b59b9-3661-499d-8640-3f40c3c89375" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5ecfa51a", - "namespace": "openrl-system", - "resourceVersion": "1792039092561663021", - "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f61ddd1e4", - "namespace": "openrl-system", - "resourceVersion": "1792039092605935011", - "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:40:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:40:41Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:40:41Z", - "name": "cold-start-fio.18de9a32081cf908", - "namespace": "openrl-system", - "resourceVersion": "1792039241446287009", - "uid": "14a62e42-13ba-4526-a04d-a934702e1163" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:18Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:18Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:18Z", - "name": "cold-start-fio.18de9a3a9867ce9f", - "namespace": "openrl-system", - "resourceVersion": "1792039278208207001", - "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad2bcac91", - "namespace": "openrl-system", - "resourceVersion": "1792039279188751014", - "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad378cd74", - "namespace": "openrl-system", - "resourceVersion": "1792039279219679006", - "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad5aed221", - "namespace": "openrl-system", - "resourceVersion": "1792039279235135013", - "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:11Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:11Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:11Z", - "name": "cold-start-fio.18de9a70cd2c6a68", - "namespace": "openrl-system", - "resourceVersion": "1792039511048479001", - "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "resourceVersion": "1791941114537823021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Stopping container api-server", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", - "namespace": "openrl-system", - "resourceVersion": "1792039545418735013", - "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "namespace": "openrl-system", - "resourceVersion": "1792039545300351007", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", - "namespace": "openrl-system", - "resourceVersion": "1792039545387567001", - "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" - }, - "reason": "SuccessfulDelete", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546048591003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", - "namespace": "openrl-system", - "resourceVersion": "1792039546102351012", - "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:49Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:49Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", - "namespace": "openrl-system", - "resourceVersion": "1792039549712335019", - "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", - "namespace": "openrl-system", - "resourceVersion": "1792039558721215017", - "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", - "namespace": "openrl-system", - "resourceVersion": "1792039558783199013", - "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", - "namespace": "openrl-system", - "resourceVersion": "1792039558862303009", - "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "namespace": "openrl-system", - "resourceVersion": "1792039545940671018", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", - "namespace": "openrl-system", - "resourceVersion": "1792039546068975009", - "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" - }, - "reason": "SuccessfulCreate", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545232655018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78c9668cf7", - "namespace": "openrl-system", - "resourceVersion": "1792039545327119005", - "uid": "7b1433df-3524-478b-aba2-8979950e245e" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545464319018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78ef8b9e1b", - "namespace": "openrl-system", - "resourceVersion": "1792039545956079020", - "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:15Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875490623001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:15Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", - "namespace": "openrl-system", - "resourceVersion": "1792039875609279013", - "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:19Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:19Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", - "namespace": "openrl-system", - "resourceVersion": "1792039879126319008", - "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", - "namespace": "openrl-system", - "resourceVersion": "1792040061645247020", - "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", - "namespace": "openrl-system", - "resourceVersion": "1792040061849391002", - "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", - "namespace": "openrl-system", - "resourceVersion": "1792040062252079023", - "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", - "namespace": "openrl-system", - "resourceVersion": "1792040192780783021", - "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264403887001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", - "namespace": "openrl-system", - "resourceVersion": "1792040264444463021", - "uid": "a85006a2-68a6-4919-b148-08d58b546d07" - }, - "reason": "FailedScheduling", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Warning" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264434463001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", - "namespace": "openrl-system", - "resourceVersion": "1792040264884223002", - "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" - }, - "reason": "NotTriggerScaleUp", - "reportingComponent": "cluster-autoscaler", - "reportingInstance": "", - "source": { - "component": "cluster-autoscaler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264893039001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", - "namespace": "openrl-system", - "resourceVersion": "1792040293171567001", - "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", - "namespace": "openrl-system", - "resourceVersion": "1792040296672127014", - "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", - "namespace": "openrl-system", - "resourceVersion": "1792040296703007017", - "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", - "namespace": "openrl-system", - "resourceVersion": "1792040296889487014", - "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", - "namespace": "openrl-system", - "resourceVersion": "1792040427147711019", - "uid": "cc4da270-def2-456f-9374-372be8c717e4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:38Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498635999001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:38Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:38Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", - "namespace": "openrl-system", - "resourceVersion": "1792040498767471000", - "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", - "namespace": "openrl-system", - "resourceVersion": "1792040502270255017", - "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", - "namespace": "openrl-system", - "resourceVersion": "1792040502296943006", - "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", - "namespace": "openrl-system", - "resourceVersion": "1792040502475407023", - "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:09Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:09Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:09Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", - "namespace": "openrl-system", - "resourceVersion": "1792040649924575012", - "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:52Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672220015012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:52Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", - "namespace": "openrl-system", - "resourceVersion": "1792039672350591019", - "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:56Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:56Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:56Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", - "namespace": "openrl-system", - "resourceVersion": "1792039676166239015", - "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", - "namespace": "openrl-system", - "resourceVersion": "1792039844457167000", - "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", - "namespace": "openrl-system", - "resourceVersion": "1792039844632591021", - "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:45Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:45Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", - "namespace": "openrl-system", - "resourceVersion": "1792039845170959016", - "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", - "namespace": "openrl-system", - "resourceVersion": "1792040192981183013", - "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:23Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040242926223012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:23Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:23Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", - "namespace": "openrl-system", - "resourceVersion": "1792040243043743024", - "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", - "namespace": "openrl-system", - "resourceVersion": "1792040246545487023", - "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", - "namespace": "openrl-system", - "resourceVersion": "1792040246576895006", - "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", - "namespace": "openrl-system", - "resourceVersion": "1792040246742815002", - "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", - "namespace": "openrl-system", - "resourceVersion": "1792040427285631023", - "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:16Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476805407012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:16Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", - "namespace": "openrl-system", - "resourceVersion": "1792040476942623014", - "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", - "namespace": "openrl-system", - "resourceVersion": "1792040480747055005", - "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", - "namespace": "openrl-system", - "resourceVersion": "1792040480787967019", - "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", - "namespace": "openrl-system", - "resourceVersion": "1792040480963103007", - "uid": "d3158913-c79e-43d4-b468-d748641d49fe" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:10Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:10Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", - "namespace": "openrl-system", - "resourceVersion": "1792040650191935010", - "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", - "namespace": "openrl-system", - "resourceVersion": "1792038835732623016", - "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:58Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818213471023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:58Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:58Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", - "namespace": "openrl-system", - "resourceVersion": "1792040818411407005", - "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:07:01Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:07:01Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:07:01Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", - "namespace": "openrl-system", - "resourceVersion": "1792040821861695012", - "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", - "namespace": "openrl-system", - "resourceVersion": "1792040908347887024", - "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", - "namespace": "openrl-system", - "resourceVersion": "1792040908367439012", - "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", - "namespace": "openrl-system", - "resourceVersion": "1792040908418671004", - "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:10:57Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:10:57Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:10:57Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bd8cbffc810", - "namespace": "openrl-system", - "resourceVersion": "1792041057212047016", - "uid": "4a67fe5e-01b5-4b84-9453-b20887271ae4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:18Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138729647023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:18Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:18Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bebcffc9f4c", - "namespace": "openrl-system", - "resourceVersion": "1792041138871999014", - "uid": "3e19d3e2-4122-4f9d-8324-8f74961c36bc" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9c8a8459", - "namespace": "openrl-system", - "resourceVersion": "1792041142294159019", - "uid": "aaaf55ef-ec50-4fa5-b6a7-28a27d836f8e" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9d08793e", - "namespace": "openrl-system", - "resourceVersion": "1792041142311631019", - "uid": "5ab187bd-863a-4013-9e2d-185b71c68ead" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9beca1f20c38", - "namespace": "openrl-system", - "resourceVersion": "1792041142412303013", - "uid": "193a6a12-3e5a-44e3-8d69-ea9d46ee4707" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", - "namespace": "openrl-system", - "resourceVersion": "1792038835415695012", - "uid": "918abc9a-1f24-4767-8928-b202482f11ed" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:59Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699253647006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:59Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:59Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", - "namespace": "openrl-system", - "resourceVersion": "1792040699411615020", - "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:05:02Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:05:02Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:05:02Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", - "namespace": "openrl-system", - "resourceVersion": "1792040702849391020", - "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", - "namespace": "openrl-system", - "resourceVersion": "1792040789347727015", - "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", - "namespace": "openrl-system", - "resourceVersion": "1792040789380863020", - "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", - "namespace": "openrl-system", - "resourceVersion": "1792040789417615006", - "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:10:57Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:10:57Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:10:57Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9bd8d38751ab", - "namespace": "openrl-system", - "resourceVersion": "1792041057319999002", - "uid": "0af06e74-e0c1-44de-a612-6fba47fb2b5c" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106258495006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:46Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:46Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be43ff3ded1", - "namespace": "openrl-system", - "resourceVersion": "1792041106381007003", - "uid": "4842f6c0-d606-444b-a970-6e9484189707" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50d27add6", - "namespace": "openrl-system", - "resourceVersion": "1792041109821407010", - "uid": "4e9749a9-e70a-453f-b51f-4d5ea7a60e86" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50daf2189", - "namespace": "openrl-system", - "resourceVersion": "1792041109840015005", - "uid": "2950a976-a558-471f-94fb-1ac680004e4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be5134afada", - "namespace": "openrl-system", - "resourceVersion": "1792041109931471020", - "uid": "d1c0b2ae-50a0-41e8-8333-441dd2250b23" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt deleted file mode 100644 index f721cff7..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt +++ /dev/null @@ -1,222 +0,0 @@ -2026-10-15T05:12:37.408864742Z INFO 10-15 05:12:37 [model.py:692] Resolved architecture: Qwen3ForCausalLM -2026-10-15T05:12:37.409029510Z INFO 10-15 05:12:37 [model.py:2030] Using max model len 8192 -2026-10-15T05:12:37.410626201Z INFO 10-15 05:12:37 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. -2026-10-15T05:12:37.411444983Z INFO 10-15 05:12:37 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T05:12:39.815686755Z WARNING 10-15 05:12:39 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T05:12:46.943669994Z (EngineCore pid=169) INFO 10-15 05:12:46 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T05:12:47.276765140Z (EngineCore pid=169) INFO 10-15 05:12:47 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_d8308236b8a9489688b723c3383c2fc7 backend=nccl -2026-10-15T05:12:47.281242229Z (EngineCore pid=169) INFO 10-15 05:12:47 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T05:12:47.281272727Z (EngineCore pid=169) INFO 10-15 05:12:47 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T05:12:47.997459494Z (EngineCore pid=169) INFO 10-15 05:12:47 [model_runner.py:396] Loading model from scratch... -2026-10-15T05:12:48.572994219Z (EngineCore pid=169) INFO 10-15 05:12:48 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T05:12:48.573332535Z (EngineCore pid=169) INFO 10-15 05:12:48 [flash_attn.py:1116] Using FlashAttention version 3 -2026-10-15T05:12:48.688784183Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T05:12:48.689006740Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T05:12:48.986411683Z (EngineCore pid=169) INFO 10-15 05:12:48 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. -2026-10-15T05:12:48.986441106Z (EngineCore pid=169) INFO 10-15 05:12:48 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T05:12:48.987162311Z (EngineCore pid=169) -Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T05:11:54.307386428Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T05:11:54.307409055Z -2026-10-15T05:11:54.511196464Z Loading base model Qwen/Qwen3-8B to cuda:0... -2026-10-15T05:11:54.620348814Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T05:11:55.488744741Z [prefetch] reading 5 file(s), 15.26 GiB, from nfs into the page cache with 8 threads -2026-10-15T05:12:05.401433205Z [prefetch] done: 15.26 GiB in 9.9 s (1576 MiB/s) -2026-10-15T05:12:07.881306965Z -Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json deleted file mode 100644 index 7a668937..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "generateName": "open-rl-api-server-6d86b498f5-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "6d86b498f5", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - } - ], - "resourceVersion": "1792039570790271003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2ns79", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:59Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:45:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.11", - "podIPs": [ - { - "ip": "10.32.3.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:45:46Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:18Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-sampler", - "uid": "996528ae-d593-4922-86b8-ea1303243615" - } - ], - "resourceVersion": "1792041143127711023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-sampler-996528ae" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "20f5c64d-1657-4c78-92ee-24fa609f498b" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-wg5z6", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-wg5z6", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:12:23Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:12:18Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:12:23Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:12:23Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:12:18Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "40Gi" - }, - "containerID": "containerd://407aa25fc65fc92e4c27342d67e1ed826acd343dc2c6f41a01d7e922266f8d3a", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:12:22Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-wg5z6", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.1.10", - "podIPs": [ - { - "ip": "10.32.1.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:12:18Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:46Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-trainer", - "uid": "e9ddbd1c-f7f3-431b-abf2-f9f7cbdac473" - } - ], - "resourceVersion": "1792041110653471006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "9b290f43-cc99-4ea2-b399-4c09027c74d4" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-n8w58", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-n8w58", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:11:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:11:46Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:11:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:11:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:11:46Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "36Gi" - }, - "containerID": "containerd://fb935263c9491ef19bf5a85a25c2c6c1f3f99095042469eaec8e63809876f6da", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:11:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-n8w58", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.4.11", - "podIPs": [ - { - "ip": "10.32.4.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:11:46Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log deleted file mode 100644 index 4eb3d4ac..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/redis-ready.log +++ /dev/null @@ -1,342 +0,0 @@ -2026-10-15T05:11:41.877+00:00 2026-10-15T05:11:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:41.878+00:00 2026-10-15T05:11:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:42.884+00:00 2026-10-15T05:11:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:42.884+00:00 2026-10-15T05:11:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:43.893+00:00 2026-10-15T05:11:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:43.893+00:00 2026-10-15T05:11:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:44.894+00:00 2026-10-15T05:11:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:44.894+00:00 2026-10-15T05:11:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:45.899+00:00 2026-10-15T05:11:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:45.899+00:00 2026-10-15T05:11:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:46.917+00:00 2026-10-15T05:11:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:46.917+00:00 2026-10-15T05:11:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:47.910+00:00 2026-10-15T05:11:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:47.910+00:00 2026-10-15T05:11:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:48.918+00:00 2026-10-15T05:11:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:48.918+00:00 2026-10-15T05:11:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:49.921+00:00 2026-10-15T05:11:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:49.922+00:00 2026-10-15T05:11:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:50.927+00:00 2026-10-15T05:11:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:50.927+00:00 2026-10-15T05:11:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:51.932+00:00 2026-10-15T05:11:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:51.932+00:00 2026-10-15T05:11:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:52.938+00:00 2026-10-15T05:11:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:52.938+00:00 2026-10-15T05:11:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:53.946+00:00 2026-10-15T05:11:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:53.946+00:00 2026-10-15T05:11:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:54.948+00:00 2026-10-15T05:11:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:54.949+00:00 2026-10-15T05:11:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:55.952+00:00 2026-10-15T05:11:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:55.954+00:00 2026-10-15T05:11:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:56.958+00:00 2026-10-15T05:11:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:56.958+00:00 2026-10-15T05:11:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:57.963+00:00 2026-10-15T05:11:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:57.963+00:00 2026-10-15T05:11:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:59.058+00:00 2026-10-15T05:11:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:59.058+00:00 2026-10-15T05:11:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:11:59.975+00:00 2026-10-15T05:11:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:11:59.975+00:00 2026-10-15T05:11:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:00.979+00:00 2026-10-15T05:12:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:00.979+00:00 2026-10-15T05:12:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:01.983+00:00 2026-10-15T05:12:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:01.983+00:00 2026-10-15T05:12:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:02.988+00:00 2026-10-15T05:12:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:02.988+00:00 2026-10-15T05:12:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:03.993+00:00 2026-10-15T05:12:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:03.993+00:00 2026-10-15T05:12:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:04.999+00:00 2026-10-15T05:12:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:04.999+00:00 2026-10-15T05:12:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:06.004+00:00 2026-10-15T05:12:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:06.004+00:00 2026-10-15T05:12:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:07.010+00:00 2026-10-15T05:12:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:07.010+00:00 2026-10-15T05:12:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:08.013+00:00 2026-10-15T05:12:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:08.013+00:00 2026-10-15T05:12:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:09.025+00:00 2026-10-15T05:12:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:09.025+00:00 2026-10-15T05:12:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:10.023+00:00 2026-10-15T05:12:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:10.023+00:00 2026-10-15T05:12:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:11.028+00:00 2026-10-15T05:12:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:11.028+00:00 2026-10-15T05:12:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:12.035+00:00 2026-10-15T05:12:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:12.035+00:00 2026-10-15T05:12:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:13.040+00:00 2026-10-15T05:12:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:13.040+00:00 2026-10-15T05:12:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:14.046+00:00 2026-10-15T05:12:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:14.046+00:00 2026-10-15T05:12:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:15.050+00:00 2026-10-15T05:12:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:15.050+00:00 2026-10-15T05:12:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:16.056+00:00 2026-10-15T05:12:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:16.056+00:00 2026-10-15T05:12:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:17.059+00:00 2026-10-15T05:12:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:17.059+00:00 2026-10-15T05:12:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:18.064+00:00 2026-10-15T05:12:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:18.064+00:00 2026-10-15T05:12:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:19.069+00:00 2026-10-15T05:12:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:19.069+00:00 2026-10-15T05:12:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:20.074+00:00 2026-10-15T05:12:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:20.074+00:00 2026-10-15T05:12:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:21.079+00:00 2026-10-15T05:12:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:21.079+00:00 2026-10-15T05:12:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:22.086+00:00 2026-10-15T05:12:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:22.086+00:00 2026-10-15T05:12:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:23.091+00:00 2026-10-15T05:12:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:23.091+00:00 2026-10-15T05:12:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:24.097+00:00 2026-10-15T05:12:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:24.097+00:00 2026-10-15T05:12:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:25.101+00:00 2026-10-15T05:12:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:25.101+00:00 2026-10-15T05:12:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:26.106+00:00 2026-10-15T05:12:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:26.106+00:00 2026-10-15T05:12:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:27.112+00:00 2026-10-15T05:12:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:27.112+00:00 2026-10-15T05:12:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:28.121+00:00 2026-10-15T05:12:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:28.121+00:00 2026-10-15T05:12:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:29.121+00:00 2026-10-15T05:12:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:29.121+00:00 2026-10-15T05:12:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:30.130+00:00 2026-10-15T05:12:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:30.131+00:00 2026-10-15T05:12:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:31.131+00:00 2026-10-15T05:12:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:31.131+00:00 2026-10-15T05:12:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:32.136+00:00 2026-10-15T05:12:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:32.136+00:00 2026-10-15T05:12:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:33.141+00:00 2026-10-15T05:12:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:33.141+00:00 2026-10-15T05:12:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:34.146+00:00 2026-10-15T05:12:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:34.146+00:00 2026-10-15T05:12:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:35.150+00:00 2026-10-15T05:12:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:35.151+00:00 2026-10-15T05:12:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:36.156+00:00 2026-10-15T05:12:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:36.156+00:00 2026-10-15T05:12:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:37.160+00:00 2026-10-15T05:12:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:37.160+00:00 2026-10-15T05:12:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:38.166+00:00 2026-10-15T05:12:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:38.166+00:00 2026-10-15T05:12:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:39.170+00:00 2026-10-15T05:12:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:39.171+00:00 2026-10-15T05:12:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:40.176+00:00 2026-10-15T05:12:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:40.177+00:00 2026-10-15T05:12:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:41.181+00:00 2026-10-15T05:12:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:41.182+00:00 2026-10-15T05:12:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:42.186+00:00 2026-10-15T05:12:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:42.186+00:00 2026-10-15T05:12:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:43.191+00:00 2026-10-15T05:12:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:43.191+00:00 2026-10-15T05:12:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:44.197+00:00 2026-10-15T05:12:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:44.197+00:00 2026-10-15T05:12:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:45.202+00:00 2026-10-15T05:12:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:45.203+00:00 2026-10-15T05:12:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:46.210+00:00 2026-10-15T05:12:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:46.210+00:00 2026-10-15T05:12:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:47.214+00:00 2026-10-15T05:12:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:47.214+00:00 2026-10-15T05:12:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:48.220+00:00 2026-10-15T05:12:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:48.220+00:00 2026-10-15T05:12:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:49.224+00:00 2026-10-15T05:12:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:49.224+00:00 2026-10-15T05:12:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:50.261+00:00 2026-10-15T05:12:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:50.261+00:00 2026-10-15T05:12:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:51.235+00:00 2026-10-15T05:12:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:51.235+00:00 2026-10-15T05:12:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:52.240+00:00 2026-10-15T05:12:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:52.240+00:00 2026-10-15T05:12:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:53.246+00:00 2026-10-15T05:12:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:53.246+00:00 2026-10-15T05:12:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:54.251+00:00 2026-10-15T05:12:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:54.251+00:00 2026-10-15T05:12:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:55.256+00:00 2026-10-15T05:12:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:55.256+00:00 2026-10-15T05:12:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:56.262+00:00 2026-10-15T05:12:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:56.262+00:00 2026-10-15T05:12:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:57.267+00:00 2026-10-15T05:12:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:57.267+00:00 2026-10-15T05:12:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:58.272+00:00 2026-10-15T05:12:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:58.272+00:00 2026-10-15T05:12:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:12:59.282+00:00 2026-10-15T05:12:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:12:59.282+00:00 2026-10-15T05:12:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:00.282+00:00 2026-10-15T05:13:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:00.282+00:00 2026-10-15T05:13:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:01.288+00:00 2026-10-15T05:13:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:01.288+00:00 2026-10-15T05:13:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:02.293+00:00 2026-10-15T05:13:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:02.293+00:00 2026-10-15T05:13:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:03.298+00:00 2026-10-15T05:13:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:03.298+00:00 2026-10-15T05:13:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:04.305+00:00 2026-10-15T05:13:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:04.305+00:00 2026-10-15T05:13:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:05.311+00:00 2026-10-15T05:13:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:05.311+00:00 2026-10-15T05:13:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:06.315+00:00 2026-10-15T05:13:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:06.315+00:00 2026-10-15T05:13:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:07.320+00:00 2026-10-15T05:13:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:07.321+00:00 2026-10-15T05:13:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:08.325+00:00 2026-10-15T05:13:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:08.325+00:00 2026-10-15T05:13:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:09.331+00:00 2026-10-15T05:13:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:09.332+00:00 2026-10-15T05:13:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:10.344+00:00 2026-10-15T05:13:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:10.344+00:00 2026-10-15T05:13:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:11.378+00:00 2026-10-15T05:13:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:11.378+00:00 2026-10-15T05:13:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:12.350+00:00 2026-10-15T05:13:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:12.350+00:00 2026-10-15T05:13:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:13.355+00:00 2026-10-15T05:13:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:13.355+00:00 2026-10-15T05:13:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:14.360+00:00 2026-10-15T05:13:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:14.360+00:00 2026-10-15T05:13:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:15.366+00:00 2026-10-15T05:13:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:15.366+00:00 2026-10-15T05:13:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:16.371+00:00 2026-10-15T05:13:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:16.371+00:00 2026-10-15T05:13:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:17.376+00:00 2026-10-15T05:13:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:17.376+00:00 2026-10-15T05:13:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:18.382+00:00 2026-10-15T05:13:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:18.382+00:00 2026-10-15T05:13:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:19.386+00:00 2026-10-15T05:13:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:19.387+00:00 2026-10-15T05:13:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:20.393+00:00 2026-10-15T05:13:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:20.393+00:00 2026-10-15T05:13:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:21.616+00:00 2026-10-15T05:13:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:21.616+00:00 2026-10-15T05:13:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:22.404+00:00 2026-10-15T05:13:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:22.404+00:00 2026-10-15T05:13:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:23.408+00:00 2026-10-15T05:13:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:23.408+00:00 2026-10-15T05:13:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:24.412+00:00 2026-10-15T05:13:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:24.413+00:00 2026-10-15T05:13:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:25.417+00:00 2026-10-15T05:13:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:25.417+00:00 2026-10-15T05:13:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:26.424+00:00 2026-10-15T05:13:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:26.424+00:00 2026-10-15T05:13:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:27.435+00:00 2026-10-15T05:13:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:27.436+00:00 2026-10-15T05:13:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:28.436+00:00 2026-10-15T05:13:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:28.436+00:00 2026-10-15T05:13:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:29.441+00:00 2026-10-15T05:13:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:29.441+00:00 2026-10-15T05:13:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:30.451+00:00 2026-10-15T05:13:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:30.451+00:00 2026-10-15T05:13:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:31.452+00:00 2026-10-15T05:13:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:31.452+00:00 2026-10-15T05:13:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:32.463+00:00 2026-10-15T05:13:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:32.463+00:00 2026-10-15T05:13:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:33.462+00:00 2026-10-15T05:13:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:33.462+00:00 2026-10-15T05:13:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:34.467+00:00 2026-10-15T05:13:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:34.467+00:00 2026-10-15T05:13:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:35.473+00:00 2026-10-15T05:13:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:35.473+00:00 2026-10-15T05:13:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:36.478+00:00 2026-10-15T05:13:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:36.478+00:00 2026-10-15T05:13:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:37.483+00:00 2026-10-15T05:13:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:37.483+00:00 2026-10-15T05:13:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:38.488+00:00 2026-10-15T05:13:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:38.489+00:00 2026-10-15T05:13:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:39.507+00:00 2026-10-15T05:13:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:39.507+00:00 2026-10-15T05:13:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:40.499+00:00 2026-10-15T05:13:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:40.499+00:00 2026-10-15T05:13:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:41.507+00:00 2026-10-15T05:13:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:41.507+00:00 2026-10-15T05:13:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:42.511+00:00 2026-10-15T05:13:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:42.511+00:00 2026-10-15T05:13:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:43.516+00:00 2026-10-15T05:13:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:43.516+00:00 2026-10-15T05:13:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:44.521+00:00 2026-10-15T05:13:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:44.521+00:00 2026-10-15T05:13:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:45.526+00:00 2026-10-15T05:13:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:45.526+00:00 2026-10-15T05:13:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:46.531+00:00 2026-10-15T05:13:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:46.532+00:00 2026-10-15T05:13:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:47.537+00:00 2026-10-15T05:13:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:47.537+00:00 2026-10-15T05:13:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:48.542+00:00 2026-10-15T05:13:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:48.542+00:00 2026-10-15T05:13:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:49.549+00:00 2026-10-15T05:13:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:49.549+00:00 2026-10-15T05:13:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:50.554+00:00 2026-10-15T05:13:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:50.554+00:00 2026-10-15T05:13:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:51.560+00:00 2026-10-15T05:13:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:51.560+00:00 2026-10-15T05:13:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:52.565+00:00 2026-10-15T05:13:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:52.565+00:00 2026-10-15T05:13:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:53.571+00:00 2026-10-15T05:13:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:53.571+00:00 2026-10-15T05:13:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:54.578+00:00 2026-10-15T05:13:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:54.578+00:00 2026-10-15T05:13:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:55.580+00:00 2026-10-15T05:13:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:55.580+00:00 2026-10-15T05:13:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:56.585+00:00 2026-10-15T05:13:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:56.586+00:00 2026-10-15T05:13:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:57.590+00:00 2026-10-15T05:13:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:57.591+00:00 2026-10-15T05:13:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:58.609+00:00 2026-10-15T05:13:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:58.609+00:00 2026-10-15T05:13:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:13:59.601+00:00 2026-10-15T05:13:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:13:59.601+00:00 2026-10-15T05:13:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:00.606+00:00 2026-10-15T05:14:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:00.607+00:00 2026-10-15T05:14:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:01.612+00:00 2026-10-15T05:14:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:01.612+00:00 2026-10-15T05:14:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:02.619+00:00 2026-10-15T05:14:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:02.619+00:00 2026-10-15T05:14:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:03.623+00:00 2026-10-15T05:14:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:03.623+00:00 2026-10-15T05:14:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:04.628+00:00 2026-10-15T05:14:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:04.628+00:00 2026-10-15T05:14:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:05.639+00:00 2026-10-15T05:14:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:05.639+00:00 2026-10-15T05:14:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:06.638+00:00 2026-10-15T05:14:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:06.639+00:00 2026-10-15T05:14:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:07.645+00:00 2026-10-15T05:14:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:07.645+00:00 2026-10-15T05:14:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:08.649+00:00 2026-10-15T05:14:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:08.649+00:00 2026-10-15T05:14:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:09.655+00:00 2026-10-15T05:14:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:09.656+00:00 2026-10-15T05:14:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:10.662+00:00 2026-10-15T05:14:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:10.662+00:00 2026-10-15T05:14:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:11.668+00:00 2026-10-15T05:14:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:11.668+00:00 2026-10-15T05:14:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:12.671+00:00 2026-10-15T05:14:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:12.671+00:00 2026-10-15T05:14:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:13.677+00:00 2026-10-15T05:14:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:13.677+00:00 2026-10-15T05:14:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:14.683+00:00 2026-10-15T05:14:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:14.684+00:00 2026-10-15T05:14:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:15.688+00:00 2026-10-15T05:14:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:15.688+00:00 2026-10-15T05:14:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:16.693+00:00 2026-10-15T05:14:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:16.693+00:00 2026-10-15T05:14:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:17.698+00:00 2026-10-15T05:14:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:17.698+00:00 2026-10-15T05:14:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:18.703+00:00 2026-10-15T05:14:18Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:18.703+00:00 2026-10-15T05:14:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:18.703+00:00 2026-10-15T05:14:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:19.707+00:00 2026-10-15T05:14:19Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:19.707+00:00 2026-10-15T05:14:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:19.707+00:00 2026-10-15T05:14:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:20.713+00:00 2026-10-15T05:14:20Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:20.713+00:00 2026-10-15T05:14:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:20.713+00:00 2026-10-15T05:14:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:21.718+00:00 2026-10-15T05:14:21Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:21.718+00:00 2026-10-15T05:14:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:21.718+00:00 2026-10-15T05:14:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:22.723+00:00 2026-10-15T05:14:22Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:22.723+00:00 2026-10-15T05:14:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:22.723+00:00 2026-10-15T05:14:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:23.729+00:00 2026-10-15T05:14:23Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:23.729+00:00 2026-10-15T05:14:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:23.729+00:00 2026-10-15T05:14:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:24.734+00:00 2026-10-15T05:14:24Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:24.735+00:00 2026-10-15T05:14:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:24.735+00:00 2026-10-15T05:14:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:25.742+00:00 2026-10-15T05:14:25Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:25.742+00:00 2026-10-15T05:14:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:25.742+00:00 2026-10-15T05:14:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:26.745+00:00 2026-10-15T05:14:26Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:26.745+00:00 2026-10-15T05:14:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:26.745+00:00 2026-10-15T05:14:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:14:27.787+00:00 2026-10-15T05:14:27Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:14:27.787+00:00 2026-10-15T05:14:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:14:27.787+00:00 2026-10-15T05:14:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log deleted file mode 100644 index d52fd557..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T05:11:46.113+00:00 2026-10-15T05:11:46.136519288Z {"level":"info","ts":"2026-10-15T05:11:46Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"76087a4e-44f6-4134-908c-44b725c30029","claim":"claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} -2026-10-15T05:12:18.598+00:00 2026-10-15T05:12:18.620658575Z {"level":"info","ts":"2026-10-15T05:12:18Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"30427539-027e-42b8-833c-14c6aa57927a","claim":"claim-lora-qwen-qwen3-8b-0-sampler-996528ae","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json deleted file mode 100644 index 279002b2..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/timeline.json +++ /dev/null @@ -1,267 +0,0 @@ -{ - "case": "after-D1-8b-h100-cached", - "collected": "2026-10-15T05:14:27.870+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 05:11:45.156000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen3-8B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 05:11:45.157000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 05:12:10.447000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "428ca04c-2ae6-41ed-9a4b-94f1cd79c97e" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 05:12:13.323000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 05:12:16.232000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 05:12:18.352000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 05:12:18.353000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 05:12:20.273000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 05:12:20.275000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 05:14:19.688000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4\n\nQuestion: What is " - }, - "client.end": { - "ts": "2026-10-15 05:14:19.689000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen3-8b-0-sampler", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "pod": "orw-lora-qwen-qwen3-8b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 05:12:18+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:12:18.877000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 05:12:18.877000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 05:12:18.877000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 05:12:23+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 05:12:18+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 05:12:18+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 05:12:23+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 05:12:23+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 05:12:18+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 05:12:22+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 05:12:18+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 05:12:22+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "sampler.event_Created": { - "ts": "2026-10-15 05:12:22+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 05:12:22+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 05:12:37.408864+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 05:12:46.943669+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:12:46 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 05:12:47.997459+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:12:47 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 05:12:48.986441+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:12:48 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 05:13:00.893700+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:00 [default_loader.py:430] Loading weights took 11.91 seconds", - "self_reported_s": 11.91 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 05:13:01.578989+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:01 [model_runner.py:428] Model loading took 18.17 GiB memory and 13.845580 seconds", - "self_reported_s": 13.84558 - }, - "sampler.compile_done": { - "ts": "2026-10-15 05:13:38.517327+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:38 [monitor.py:53] torch.compile took 35.91 s in total", - "self_reported_s": 35.91 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 05:13:58.672379+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:58 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", - "self_reported_s": 16.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 05:13:59.319915+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:13:59 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 05:14:17.795200+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:14:17 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.21 s", - "self_reported_s": 76.21 - }, - "sampler.ready_key": { - "ts": "2026-10-15 05:14:18+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - }, - "trainer": { - "workload": "lora-qwen-qwen3-8b-0-trainer", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "pod": "orw-lora-qwen-qwen3-8b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 05:11:45+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:11:45.752000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 05:11:45.752000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 05:11:47.402000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 05:11:50+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 05:11:46+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 05:11:46+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 05:11:50+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 05:11:50+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 05:11:46+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 05:11:49+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 05:11:46+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 05:11:49+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "trainer.event_Created": { - "ts": "2026-10-15 05:11:49+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 05:11:49+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 05:11:54.291598+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 05:11:54.511196+00:00", - "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 05:12:08.031266+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 05:12:08.033054+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 05:11:41+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt deleted file mode 100644 index b0d2d90a..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:11:40.798+00:00 diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt deleted file mode 100644 index a9917932..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json b/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json deleted file mode 100644 index 467556c3..00000000 --- a/runs/2026-10-15_cold_start/after-D1-8b-h100-cached/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:18Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041143235791008", - "uid": "996528ae-d593-4922-86b8-ea1303243615" - }, - "spec": { - "accelerator": { - "memory": "29Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "20f5c64d-1657-4c78-92ee-24fa609f498b", - "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-996528ae", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:12:23Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-996528ae", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "29Gi", - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:45Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041110758783004", - "uid": "e9ddbd1c-f7f3-431b-abf2-f9f7cbdac473" - }, - "spec": { - "accelerator": { - "memory": "20Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "9b290f43-cc99-4ea2-b399-4c09027c74d4", - "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:11:50Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-e9ddbd1c", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "20Gi", - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log deleted file mode 100644 index 3fc7c971..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached.run.log +++ /dev/null @@ -1,35 +0,0 @@ -watching; recorders [19286, 19287, 19289, 19291, 19293, 19288, 19290, 19292] write into runs/2026-10-15_cold_start/after-D2-8b-h100-cached -collected 2 worker log(s) into runs/2026-10-15_cold_start/after-D2-8b-h100-cached -## Critical path (derived from the marks above) - -### trainer - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.create_model.start | trainer.pod_created | 0.4 | -| pod scheduled | trainer.pod_created | trainer.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | trainer.pod_PodScheduled | trainer.event_Pulled | 4.0 | -| container create | trainer.event_Pulled | trainer.container_started | 0.0 | -| process start to load start (imports) | trainer.container_started | trainer.load_start | 5.1 | -| weights load | trainer.load_start | trainer.load_done | 12.1 | -| ready key | trainer.load_done | trainer.ready_key | -24.3 | -| client: create_model blocked | client.create_model.start | client.create_model.done | 24.1 | -| client: first forward_backward | client.forward_backward.start | client.forward_backward.done | 2.9 | - -### sampler - -| Phase | From | To | Seconds | -|---|---|---|---| -| placement: Workload to pod | client.save_weights_and_get_sampling_client.start | sampler.pod_created | 0.1 | -| pod scheduled | sampler.pod_created | sampler.pod_PodScheduled | 0.0 | -| image pull (0 if cached) | sampler.pod_PodScheduled | sampler.event_Pulled | 4.0 | -| container create | sampler.event_Pulled | sampler.container_started | 0.0 | -| process start to first log | sampler.container_started | sampler.process_first_log | 15.5 | -| imports until engine init | sampler.process_first_log | sampler.engine_init_start | 9.7 | -| engine init until load start | sampler.engine_init_start | sampler.load_start | 1.1 | -| weights load (vLLM prefetch into page cache + load) | sampler.load_start | sampler.weights_loaded | 13.1 | -| torch.compile | sampler.weights_loaded | sampler.compile_done | 37.2 | -| profiling, KV cache, graph capture | sampler.compile_done | sampler.engine_init_done | 39.5 | -| ready key | sampler.engine_init_done | sampler.ready_key | 0.9 | -| client: first sample blocked | client.sample.start | client.sample.done | 119.9 | - diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log deleted file mode 100644 index b4c54e58..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/api-server.log +++ /dev/null @@ -1,32 +0,0 @@ -2026-10-15T05:15:36.893+00:00 2026-10-15T05:15:36.893863054Z INFO: 127.0.0.1:35824 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T05:15:37.116+00:00 2026-10-15T05:15:37.118777792Z INFO: 127.0.0.1:35826 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T05:15:37.193+00:00 2026-10-15T05:15:37.194477785Z INFO: 127.0.0.1:35826 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T05:15:37.636+00:00 2026-10-15T05:15:37.638416243Z [API_SERVER] enqueued op=create_model request_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:15:37.637+00:00 2026-10-15T05:15:37.639053003Z INFO: 127.0.0.1:35832 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T05:15:40.746+00:00 2026-10-15T05:15:40.746550012Z INFO: 10.32.3.1:39766 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:15:47.379+00:00 2026-10-15T05:15:47.379417038Z INFO: 127.0.0.1:54214 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:15:50.746+00:00 2026-10-15T05:15:50.746722382Z INFO: 10.32.3.1:56336 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:16:00.752+00:00 2026-10-15T05:16:00.747547185Z INFO: 10.32.3.1:37336 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:16:00.959+00:00 2026-10-15T05:16:00.949527284Z INFO: 127.0.0.1:33956 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T05:16:03.946+00:00 2026-10-15T05:16:03.946007674Z [API_SERVER] enqueued op=forward_backward request_id=f37e31ba-ce81-44ac-8856-bfda6aa3c7e1 model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:16:03.946+00:00 2026-10-15T05:16:03.946615325Z INFO: 127.0.0.1:33956 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T05:16:06.905+00:00 2026-10-15T05:16:06.905826356Z [API_SERVER] enqueued op=optim_step request_id=957669d4-dc27-4c81-837d-2a8ecf9bb2f4 model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:16:06.905+00:00 2026-10-15T05:16:06.907396223Z INFO: 127.0.0.1:33956 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T05:16:09.100+00:00 2026-10-15T05:16:09.049018149Z [API_SERVER] enqueued op=save_weights_for_sampler request_id=3209fcb1-7f4e-446c-95c6-94a7e30b047f model_id=ce0e7790-7ff6-44f1-9632-06939e7b1053 active_set=Qwen/Qwen3-8B-1 -2026-10-15T05:16:09.100+00:00 2026-10-15T05:16:09.049557898Z INFO: 127.0.0.1:33956 - "POST /api/v1/save_weights_for_sampler HTTP/1.1" 200 OK -2026-10-15T05:16:10.931+00:00 2026-10-15T05:16:10.746565689Z INFO: 10.32.3.1:45574 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:16:11.296+00:00 2026-10-15T05:16:11.293332808Z INFO: 127.0.0.1:36994 - "POST /api/v1/asample HTTP/1.1" 200 OK -2026-10-15T05:16:20.765+00:00 2026-10-15T05:16:20.746229306Z INFO: 10.32.3.1:38344 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:16:30.748+00:00 2026-10-15T05:16:30.746892251Z INFO: 10.32.3.1:54454 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:16:40.748+00:00 2026-10-15T05:16:40.747033006Z INFO: 10.32.3.1:59170 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:16:50.747+00:00 2026-10-15T05:16:50.747504698Z INFO: 10.32.3.1:43604 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:16:57.561+00:00 2026-10-15T05:16:57.560755551Z INFO: 127.0.0.1:36756 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:17:00.747+00:00 2026-10-15T05:17:00.747501058Z INFO: 10.32.3.1:45120 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:17:10.747+00:00 2026-10-15T05:17:10.746700462Z INFO: 10.32.3.1:46882 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:17:20.747+00:00 2026-10-15T05:17:20.747102135Z INFO: 10.32.3.1:37964 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:17:30.751+00:00 2026-10-15T05:17:30.746961545Z INFO: 10.32.3.1:40486 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:17:40.748+00:00 2026-10-15T05:17:40.747075210Z INFO: 10.32.3.1:48770 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:17:47.748+00:00 2026-10-15T05:17:47.748538952Z INFO: 127.0.0.1:47530 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:17:50.747+00:00 2026-10-15T05:17:50.746643982Z INFO: 10.32.3.1:50180 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:18:00.747+00:00 2026-10-15T05:18:00.746678248Z INFO: 10.32.3.1:38072 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:18:10.746+00:00 2026-10-15T05:18:10.746445484Z INFO: 10.32.3.1:42070 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl deleted file mode 100644 index dfbb3c97..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"ts": "2026-10-15T05:15:36.633+00:00", "t": 0.0, "phase": "start", "base_model": "Qwen/Qwen3-8B", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T05:15:36.633+00:00", "t": 0.001, "phase": "create_model.start"} -{"ts": "2026-10-15T05:16:00.765+00:00", "t": 24.133, "phase": "create_model.done", "model_id": "ce0e7790-7ff6-44f1-9632-06939e7b1053"} -{"ts": "2026-10-15T05:16:03.882+00:00", "t": 27.25, "phase": "forward_backward.start"} -{"ts": "2026-10-15T05:16:06.825+00:00", "t": 30.192, "phase": "forward_backward.done"} -{"ts": "2026-10-15T05:16:08.923+00:00", "t": 32.29, "phase": "optim_step.done"} -{"ts": "2026-10-15T05:16:08.924+00:00", "t": 32.291, "phase": "save_weights_and_get_sampling_client.start"} -{"ts": "2026-10-15T05:16:11.071+00:00", "t": 34.438, "phase": "save_weights_and_get_sampling_client.done"} -{"ts": "2026-10-15T05:16:11.076+00:00", "t": 34.443, "phase": "sample.start"} -{"ts": "2026-10-15T05:18:10.965+00:00", "t": 154.332, "phase": "sample.done", "text": " 4\n\nQuestion: What is "} -{"ts": "2026-10-15T05:18:10.966+00:00", "t": 154.334, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log deleted file mode 100644 index fedb02d3..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/client.log +++ /dev/null @@ -1,12 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'Qwen/Qwen3-8B', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.001s create_model.start -[cold-start] 24.133s create_model.done {'model_id': 'ce0e7790-7ff6-44f1-9632-06939e7b1053'} -[cold-start] 27.250s forward_backward.start -[cold-start] 30.192s forward_backward.done -[cold-start] 32.290s optim_step.done -[cold-start] 32.291s save_weights_and_get_sampling_client.start -[cold-start] 34.438s save_weights_and_get_sampling_client.done -[cold-start] 34.443s sample.start -[cold-start] 154.332s sample.done {'text': ' 4\n\nQuestion: What is '} -[cold-start] 154.334s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt deleted file mode 100644 index 9458726e..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:18:11.482+00:00 diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json deleted file mode 100644 index 6a2fe366..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/events.json +++ /dev/null @@ -1,3069 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:10Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:10Z", - "name": "cold-start-fio.18de9a0ef295773d", - "namespace": "openrl-system", - "resourceVersion": "1792039090760383011", - "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5be63862", - "namespace": "openrl-system", - "resourceVersion": "1792039092515007000", - "uid": "355b59b9-3661-499d-8640-3f40c3c89375" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5ecfa51a", - "namespace": "openrl-system", - "resourceVersion": "1792039092561663021", - "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f61ddd1e4", - "namespace": "openrl-system", - "resourceVersion": "1792039092605935011", - "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:40:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:40:41Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:40:41Z", - "name": "cold-start-fio.18de9a32081cf908", - "namespace": "openrl-system", - "resourceVersion": "1792039241446287009", - "uid": "14a62e42-13ba-4526-a04d-a934702e1163" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:18Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:18Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:18Z", - "name": "cold-start-fio.18de9a3a9867ce9f", - "namespace": "openrl-system", - "resourceVersion": "1792039278208207001", - "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad2bcac91", - "namespace": "openrl-system", - "resourceVersion": "1792039279188751014", - "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad378cd74", - "namespace": "openrl-system", - "resourceVersion": "1792039279219679006", - "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad5aed221", - "namespace": "openrl-system", - "resourceVersion": "1792039279235135013", - "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:11Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:11Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:11Z", - "name": "cold-start-fio.18de9a70cd2c6a68", - "namespace": "openrl-system", - "resourceVersion": "1792039511048479001", - "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "resourceVersion": "1791941114537823021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Stopping container api-server", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", - "namespace": "openrl-system", - "resourceVersion": "1792039545418735013", - "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "namespace": "openrl-system", - "resourceVersion": "1792039545300351007", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", - "namespace": "openrl-system", - "resourceVersion": "1792039545387567001", - "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" - }, - "reason": "SuccessfulDelete", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546048591003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", - "namespace": "openrl-system", - "resourceVersion": "1792039546102351012", - "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:49Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:49Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", - "namespace": "openrl-system", - "resourceVersion": "1792039549712335019", - "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", - "namespace": "openrl-system", - "resourceVersion": "1792039558721215017", - "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", - "namespace": "openrl-system", - "resourceVersion": "1792039558783199013", - "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", - "namespace": "openrl-system", - "resourceVersion": "1792039558862303009", - "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "namespace": "openrl-system", - "resourceVersion": "1792039545940671018", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", - "namespace": "openrl-system", - "resourceVersion": "1792039546068975009", - "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" - }, - "reason": "SuccessfulCreate", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545232655018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78c9668cf7", - "namespace": "openrl-system", - "resourceVersion": "1792039545327119005", - "uid": "7b1433df-3524-478b-aba2-8979950e245e" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545464319018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78ef8b9e1b", - "namespace": "openrl-system", - "resourceVersion": "1792039545956079020", - "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:15Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875490623001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:15Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", - "namespace": "openrl-system", - "resourceVersion": "1792039875609279013", - "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:19Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:19Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", - "namespace": "openrl-system", - "resourceVersion": "1792039879126319008", - "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", - "namespace": "openrl-system", - "resourceVersion": "1792040061645247020", - "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", - "namespace": "openrl-system", - "resourceVersion": "1792040061849391002", - "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", - "namespace": "openrl-system", - "resourceVersion": "1792040062252079023", - "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", - "namespace": "openrl-system", - "resourceVersion": "1792040192780783021", - "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264403887001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", - "namespace": "openrl-system", - "resourceVersion": "1792040264444463021", - "uid": "a85006a2-68a6-4919-b148-08d58b546d07" - }, - "reason": "FailedScheduling", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Warning" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264434463001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", - "namespace": "openrl-system", - "resourceVersion": "1792040264884223002", - "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" - }, - "reason": "NotTriggerScaleUp", - "reportingComponent": "cluster-autoscaler", - "reportingInstance": "", - "source": { - "component": "cluster-autoscaler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264893039001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", - "namespace": "openrl-system", - "resourceVersion": "1792040293171567001", - "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", - "namespace": "openrl-system", - "resourceVersion": "1792040296672127014", - "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", - "namespace": "openrl-system", - "resourceVersion": "1792040296703007017", - "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", - "namespace": "openrl-system", - "resourceVersion": "1792040296889487014", - "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", - "namespace": "openrl-system", - "resourceVersion": "1792040427147711019", - "uid": "cc4da270-def2-456f-9374-372be8c717e4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:38Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498635999001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:38Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:38Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", - "namespace": "openrl-system", - "resourceVersion": "1792040498767471000", - "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", - "namespace": "openrl-system", - "resourceVersion": "1792040502270255017", - "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", - "namespace": "openrl-system", - "resourceVersion": "1792040502296943006", - "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", - "namespace": "openrl-system", - "resourceVersion": "1792040502475407023", - "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:09Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:09Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:09Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", - "namespace": "openrl-system", - "resourceVersion": "1792040649924575012", - "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:52Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672220015012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:52Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", - "namespace": "openrl-system", - "resourceVersion": "1792039672350591019", - "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:56Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:56Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:56Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", - "namespace": "openrl-system", - "resourceVersion": "1792039676166239015", - "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", - "namespace": "openrl-system", - "resourceVersion": "1792039844457167000", - "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", - "namespace": "openrl-system", - "resourceVersion": "1792039844632591021", - "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:45Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:45Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", - "namespace": "openrl-system", - "resourceVersion": "1792039845170959016", - "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", - "namespace": "openrl-system", - "resourceVersion": "1792040192981183013", - "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:23Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040242926223012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:23Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:23Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", - "namespace": "openrl-system", - "resourceVersion": "1792040243043743024", - "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", - "namespace": "openrl-system", - "resourceVersion": "1792040246545487023", - "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", - "namespace": "openrl-system", - "resourceVersion": "1792040246576895006", - "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", - "namespace": "openrl-system", - "resourceVersion": "1792040246742815002", - "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", - "namespace": "openrl-system", - "resourceVersion": "1792040427285631023", - "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:16Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476805407012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:16Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", - "namespace": "openrl-system", - "resourceVersion": "1792040476942623014", - "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", - "namespace": "openrl-system", - "resourceVersion": "1792040480747055005", - "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", - "namespace": "openrl-system", - "resourceVersion": "1792040480787967019", - "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", - "namespace": "openrl-system", - "resourceVersion": "1792040480963103007", - "uid": "d3158913-c79e-43d4-b468-d748641d49fe" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:10Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:10Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", - "namespace": "openrl-system", - "resourceVersion": "1792040650191935010", - "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", - "namespace": "openrl-system", - "resourceVersion": "1792038835732623016", - "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:58Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818213471023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:58Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:58Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", - "namespace": "openrl-system", - "resourceVersion": "1792040818411407005", - "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:07:01Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:07:01Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:07:01Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", - "namespace": "openrl-system", - "resourceVersion": "1792040821861695012", - "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", - "namespace": "openrl-system", - "resourceVersion": "1792040908347887024", - "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", - "namespace": "openrl-system", - "resourceVersion": "1792040908367439012", - "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", - "namespace": "openrl-system", - "resourceVersion": "1792040908418671004", - "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:10:57Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:10:57Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:10:57Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bd8cbffc810", - "namespace": "openrl-system", - "resourceVersion": "1792041057212047016", - "uid": "4a67fe5e-01b5-4b84-9453-b20887271ae4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:18Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138729647023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:18Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:18Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bebcffc9f4c", - "namespace": "openrl-system", - "resourceVersion": "1792041138871999014", - "uid": "3e19d3e2-4122-4f9d-8324-8f74961c36bc" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9c8a8459", - "namespace": "openrl-system", - "resourceVersion": "1792041142294159019", - "uid": "aaaf55ef-ec50-4fa5-b6a7-28a27d836f8e" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9d08793e", - "namespace": "openrl-system", - "resourceVersion": "1792041142311631019", - "uid": "5ab187bd-863a-4013-9e2d-185b71c68ead" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9beca1f20c38", - "namespace": "openrl-system", - "resourceVersion": "1792041142412303013", - "uid": "193a6a12-3e5a-44e3-8d69-ea9d46ee4707" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:14:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:14:49Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:14:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c0ec69a2183", - "namespace": "openrl-system", - "resourceVersion": "1792041289046591010", - "uid": "7161d68e-6e82-48a1-894f-7f149cadb6db" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:09Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369352463023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:09Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:09Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2181d762b3", - "namespace": "openrl-system", - "resourceVersion": "1792041369496943024", - "uid": "6a40425d-424d-438f-b398-45410b5817eb" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:13Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369459823023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:13Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:13Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c22606a262a", - "namespace": "openrl-system", - "resourceVersion": "1792041373214223010", - "uid": "c67940e2-268e-4a9a-94d4-05be2058728b" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:13Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369459823023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:13Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:13Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2260e74a63", - "namespace": "openrl-system", - "resourceVersion": "1792041373233151005", - "uid": "ecec41f0-1307-4892-ad43-57226155574a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:13Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369459823023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:13Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:13Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2266080366", - "namespace": "openrl-system", - "resourceVersion": "1792041373308543020", - "uid": "6dc932f6-ae1f-48f6-b85d-e087521c8e20" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", - "namespace": "openrl-system", - "resourceVersion": "1792038835415695012", - "uid": "918abc9a-1f24-4767-8928-b202482f11ed" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:59Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699253647006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:59Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:59Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", - "namespace": "openrl-system", - "resourceVersion": "1792040699411615020", - "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:05:02Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:05:02Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:05:02Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", - "namespace": "openrl-system", - "resourceVersion": "1792040702849391020", - "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", - "namespace": "openrl-system", - "resourceVersion": "1792040789347727015", - "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", - "namespace": "openrl-system", - "resourceVersion": "1792040789380863020", - "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", - "namespace": "openrl-system", - "resourceVersion": "1792040789417615006", - "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:10:57Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:10:57Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:10:57Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9bd8d38751ab", - "namespace": "openrl-system", - "resourceVersion": "1792041057319999002", - "uid": "0af06e74-e0c1-44de-a612-6fba47fb2b5c" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106258495006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:46Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:46Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be43ff3ded1", - "namespace": "openrl-system", - "resourceVersion": "1792041106381007003", - "uid": "4842f6c0-d606-444b-a970-6e9484189707" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50d27add6", - "namespace": "openrl-system", - "resourceVersion": "1792041109821407010", - "uid": "4e9749a9-e70a-453f-b51f-4d5ea7a60e86" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50daf2189", - "namespace": "openrl-system", - "resourceVersion": "1792041109840015005", - "uid": "2950a976-a558-471f-94fb-1ac680004e4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be5134afada", - "namespace": "openrl-system", - "resourceVersion": "1792041109931471020", - "uid": "d1c0b2ae-50a0-41e8-8333-441dd2250b23" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:14:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:14:49Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:14:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c0ece29f9b7", - "namespace": "openrl-system", - "resourceVersion": "1792041289153695008", - "uid": "9db24059-5412-4998-8f1e-ff4723a02422" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:37Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337881599006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:37Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:37Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1a2ddca97f", - "namespace": "openrl-system", - "resourceVersion": "1792041338007727010", - "uid": "622520b7-a0c6-4899-9d8e-8b7e289a5c19" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337986559006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:41Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:41Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afae88647", - "namespace": "openrl-system", - "resourceVersion": "1792041341445023014", - "uid": "174539ca-68ae-411a-9602-b94910a4f8a1" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337986559006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:41Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:41Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afb72b4d7", - "namespace": "openrl-system", - "resourceVersion": "1792041341467327017", - "uid": "abc887ed-fbc2-4086-9892-21b9dbd9ef25" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337986559006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:41Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:41Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1b0091bf79", - "namespace": "openrl-system", - "resourceVersion": "1792041341538399007", - "uid": "ce445484-5b88-4ca9-a93d-806ea29b103f" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt deleted file mode 100644 index 2a05dc10..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/log-orw-lora-qwen-qwen3-8b-0-sampler.txt +++ /dev/null @@ -1,222 +0,0 @@ -2026-10-15T05:16:28.490961146Z INFO 10-15 05:16:28 [model.py:692] Resolved architecture: Qwen3ForCausalLM -2026-10-15T05:16:28.491280812Z INFO 10-15 05:16:28 [model.py:2030] Using max model len 8192 -2026-10-15T05:16:28.492938929Z INFO 10-15 05:16:28 [scheduler.py:288] Chunked prefill is enabled with max_num_batched_tokens=16384. -2026-10-15T05:16:28.493837169Z INFO 10-15 05:16:28 [kernel.py:416] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) -2026-10-15T05:16:31.024763096Z WARNING 10-15 05:16:31 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized -2026-10-15T05:16:38.151495536Z (EngineCore pid=169) INFO 10-15 05:16:38 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Qwen3-8B', speculative_config=None, tokenizer='Qwen/Qwen3-8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=Qwen/Qwen3-8B, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) -2026-10-15T05:16:38.483358659Z (EngineCore pid=169) INFO 10-15 05:16:38 [parallel_state.py:1827] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_4d2a060e6dcc492b933d56b85af7d4e7 backend=nccl -2026-10-15T05:16:38.487228505Z (EngineCore pid=169) INFO 10-15 05:16:38 [parallel_state.py:2267] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A -2026-10-15T05:16:38.487263057Z (EngineCore pid=169) INFO 10-15 05:16:38 [gpu_worker.py:441] Using V2 Model Runner -2026-10-15T05:16:39.244394278Z (EngineCore pid=169) INFO 10-15 05:16:39 [model_runner.py:396] Loading model from scratch... -2026-10-15T05:16:39.815914136Z (EngineCore pid=169) INFO 10-15 05:16:39 [cuda.py:538] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. -2026-10-15T05:16:39.816213542Z (EngineCore pid=169) INFO 10-15 05:16:39 [flash_attn.py:1116] Using FlashAttention version 3 -2026-10-15T05:16:39.931406255Z (EngineCore pid=169) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. -2026-10-15T05:16:39.931446653Z (EngineCore pid=169) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. -2026-10-15T05:16:40.237737879Z (EngineCore pid=169) INFO 10-15 05:16:40 [weight_utils.py:895] Filesystem type for checkpoints: NFS. Checkpoint size: 15.26 GiB. Available RAM: 37.53 GiB. -2026-10-15T05:16:40.237765322Z (EngineCore pid=169) INFO 10-15 05:16:40 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes) -2026-10-15T05:16:40.238500999Z (EngineCore pid=169) -Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T05:15:45.904087714Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T05:15:45.904110382Z -2026-10-15T05:15:46.108440892Z Loading base model Qwen/Qwen3-8B to cuda:0... -2026-10-15T05:15:46.235847013Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T05:15:47.098453469Z [prefetch] reading 5 file(s), 15.26 GiB, from nfs into the page cache with 8 threads -2026-10-15T05:15:55.597395640Z [prefetch] done: 15.26 GiB in 8.5 s (1838 MiB/s) -2026-10-15T05:15:58.104601379Z -Loading weights: 0%| | 0/399 [00:00 9d v1.35.7-gke.1222000 true -gke-open-rl-dra-h100-sampler-a721f127-ed3t Ready 14d v1.35.7-gke.1222000 false -gke-open-rl-dra-h100-trainer-702095b4-1g3a Ready 26h v1.35.8-gke.1380001 true -gke-open-rl-dra-h100-trainer-702095b4-tuqu Ready 26h v1.35.8-gke.1380001 false -gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208 Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c Ready 24d v1.35.7-gke.1222000 false -gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8 Ready 24d v1.35.7-gke.1222000 false diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json deleted file mode 100644 index 2f350639..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/pods.json +++ /dev/null @@ -1,6782 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:12:28Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-66wh7", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764209195247009", - "uid": "900198f2-ba3a-4af8-8fc8-a73c4af1c45d" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-cwdlp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0e27db869c45955b79804657dc3af6c194008cafc72a64a1acbf345cf69747aa", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:16:49Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-cwdlp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:12:28Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-cm7x6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945508482719021", - "uid": "0cf34645-9d98-4c62-a9f5-de7d8bccf2b9" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-zbkzx", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:28Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://31feb315f31721f5756689a74bde20205183ae8cc3ffe1e5cb47f96a1c485b6b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:27Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-zbkzx", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:19:20Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-kpp7j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791945505996447017", - "uid": "7e2ac944-3783-438d-bb90-7a2f0c0c3b4b" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-v9zg5", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:38:25Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://708b908acebdc3de3978320d8be5f5db2297ead93495f7ac5ac1ea5797ac0e35", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:38:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-v9zg5", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:36:04Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-rz7fw", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762451790735011", - "uid": "84c03e1f-e4b6-41f9-9f21-01f8929470c2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-mrg4h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:31Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://92ff7214a9423f362487d1815eb7310817b33217c03df6a6a0090f081ef33d4b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:47:31Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mrg4h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:36:04Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:47:32Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-t99wd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791762969945423016", - "uid": "24ddf14d-a173-4360-b179-064929dbff94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-j7fh2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:56:09Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:47:32Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://a493064396312e2617fc7ce361f853fc3a990b70cd7bd197a7439105d635c79b", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:56:09Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-j7fh2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:47:32Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:03:54Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-accel-timeslicer-vlvfv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791763946154111000", - "uid": "bab0f25b-d268-4e89-91f5-d882fec173ed" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9c6fw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:12:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:03:54Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://0be5a113e58f1d262baa2e7714bc811f8a2b7adf0303a19e5c569564dab51cf1", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:12:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9c6fw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:03:54Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-12T00:16:49Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-vvwn6", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791764758862255017", - "uid": "d62cd1a5-7f1c-4121-acce-1153d3b03e60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-rfs22", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:25:58Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-12T00:16:49Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://d04cf3bd448504809a4a4b454e85c3509d35ee36478560d7cbbe1a555bbaa7a4", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-12T00:25:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-rfs22", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-12T00:16:49Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-11T23:26:48Z", - "generateName": "open-rl-accel-timeslicer-", - "generation": 1, - "labels": { - "app": "open-rl-accel-timeslicer", - "controller-revision-hash": "74748679dc", - "pod-template-generation": "3", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "open-rl-accel-timeslicer-xqx9j", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "open-rl-accel-timeslicer", - "uid": "180e97bf-4415-47ff-9278-ca88d85075cf" - } - ], - "resourceVersion": "1791761764168719003", - "uid": "1b4d7707-110e-46db-84de-c07d21826f92" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--listen-host", - "0.0.0.0", - "--port", - "9753", - "--backend", - "llmd", - "--llmd-snapshot-endpoint", - "127.0.0.1:9001", - "--scheduling-policy", - "lrs" - ], - "command": [ - "uv", - "run", - "python", - "-m", - "accel_timeslicer.serve" - ], - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "name": "accel-timeslicer", - "ports": [ - { - "containerPort": 9753, - "hostPort": 9753, - "name": "snapshot", - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirstWithHostNet", - "enableServiceLinks": true, - "hostNetwork": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 1000000, - "priorityClassName": "openrl-node-daemon", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "kube-api-access-9tztl", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:36:04Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-11T23:26:48Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "250m", - "memory": "256Mi" - }, - "containerID": "containerd://2fdc26968e42642e0405b68804d696215995102f09f1fb3c69f48c7a6101193f", - "image": "ghcr.io/gke-labs/open-rl/server:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/server@sha256:df79252c079400c0862cd172aeaa5491c260c49e60b65d56c24b360971c3aeec", - "lastState": {}, - "name": "accel-timeslicer", - "ready": true, - "resources": { - "limits": { - "memory": "1Gi" - }, - "requests": { - "cpu": "250m", - "memory": "256Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-11T23:36:03Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-9tztl", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-11T23:26:48Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "generateName": "open-rl-api-server-6d86b498f5-", - "generation": 1, - "labels": { - "app": "open-rl-api-server", - "pod-template-hash": "6d86b498f5", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - } - ], - "resourceVersion": "1792039570790271003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "spec": { - "containers": [ - { - "command": [ - "uv", - "run", - "uvicorn", - "server.api_server:app", - "--host", - "0.0.0.0", - "--port", - "8000" - ], - "env": [ - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "true" - }, - { - "name": "ENABLE_GCP_TRACE", - "valueFrom": { - "configMapKeyRef": { - "key": "ENABLE_GCP_TRACE", - "name": "open-rl-config" - } - } - }, - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "BASE_MODEL", - "valueFrom": { - "configMapKeyRef": { - "key": "BASE_MODEL", - "name": "open-rl-config" - } - } - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "SAMPLING_BACKEND", - "value": "vllm" - }, - { - "name": "OPEN_RL_WORKER_MANAGER", - "value": "scheduler" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_WORKER_IMAGE", - "valueFrom": { - "configMapKeyRef": { - "key": "worker", - "name": "open-rl-images" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "api-server", - "ports": [ - { - "containerPort": 8000, - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 30, - "httpGet": { - "path": "/api/v1/healthz", - "port": 8000, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 5 - }, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-2ns79", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:59Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:46:10Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T04:45:46Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "500m", - "memory": "1Gi" - }, - "containerID": "containerd://9d4ff1c8b4d2b04081c194684d2d3d5448979f1b9bd6bc973dc0575d5d558846", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server@sha256:bfca2a106deada9c28763a16007ac4803f31ca02d30e9e87b1f2d019865a6c66", - "lastState": {}, - "name": "api-server", - "ready": true, - "resources": { - "limits": { - "memory": "2Gi" - }, - "requests": { - "cpu": "500m", - "memory": "1Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T04:45:58Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-2ns79", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.11", - "podIPs": [ - { - "ip": "10.32.3.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T04:45:46Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:25:14Z", - "generateName": "open-rl-scheduler-7cf9d4bc47-", - "generation": 1, - "labels": { - "app": "open-rl-scheduler", - "pod-template-hash": "7cf9d4bc47", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "open-rl-scheduler-7cf9d4bc47-jtsrl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "open-rl-scheduler-7cf9d4bc47", - "uid": "d943141e-ff09-4f94-9b1b-870d28c23dd0" - } - ], - "resourceVersion": "1791941144533087007", - "uid": "56193bb8-a36c-4ad7-81b5-ff1d7c2ef0ba" - }, - "spec": { - "containers": [ - { - "args": [ - "--leader-elect", - "--health-probe-bind-address=:8081", - "--metrics-bind-address=:8080" - ], - "env": [ - { - "name": "OPEN_RL_PLACEMENT_TIMEOUT", - "value": "0" - }, - { - "name": "OPEN_RL_WORKER_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - }, - { - "name": "OPEN_RL_RECONCILE_INTERVAL", - "value": "10s" - }, - { - "name": "OPEN_RL_DEVICE_CLASS", - "value": "gpu.nvidia.com" - }, - { - "name": "OPEN_RL_DEVICE_DRIVER", - "value": "gpu.nvidia.com" - } - ], - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imagePullPolicy": "IfNotPresent", - "livenessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/healthz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 15, - "periodSeconds": 20, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "name": "manager", - "ports": [ - { - "containerPort": 8080, - "name": "metrics", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 3, - "httpGet": { - "path": "/readyz", - "port": 8081, - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": [ - "ALL" - ] - }, - "readOnlyRootFilesystem": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": { - "runAsNonRoot": true - }, - "serviceAccount": "open-rl-sa", - "serviceAccountName": "open-rl-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-kwbhw", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:44Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:25:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "100m", - "memory": "128Mi" - }, - "containerID": "containerd://4c48a59a0bffcf3176ecea7a0646b10bd68db0eff4f22218d3d3365cec1fb735", - "image": "ghcr.io/gke-labs/open-rl/scheduler:v0.0.6", - "imageID": "ghcr.io/gke-labs/open-rl/scheduler@sha256:c07b638d5b71c89de7df9a578b138e09fd334004a73c7310bd726e062ca13d73", - "lastState": {}, - "name": "manager", - "ready": true, - "resources": { - "limits": { - "memory": "512Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:25:32Z" - } - }, - "user": { - "linux": { - "gid": 65532, - "supplementalGroups": [ - 65532 - ], - "uid": 65532 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-kwbhw", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.214", - "hostIPs": [ - { - "ip": "10.128.15.214" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.3.9", - "podIPs": [ - { - "ip": "10.32.3.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:25:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:09Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-sampler-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", - "openrl.io/role": "sampler", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-sampler", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-sampler", - "uid": "0c46d0b6-726c-4452-a321-7e87b66cd63b" - } - ], - "resourceVersion": "1792041373676703023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/sampler", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-sampler-0c46d0b6" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "35cbcea6-3b99-4526-8bc3-c6a94bb35da6" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-vpgx4", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-vpgx4", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:16:13Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:16:09Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:16:13Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:16:13Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:16:09Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "40Gi" - }, - "containerID": "containerd://ca45366dcf496a89394a485c3d9c32bb5a3e61633815714b2cbba49f4e1c81e6", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:16:13Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-vpgx4", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.1.11", - "podIPs": [ - { - "ip": "10.32.1.11" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:16:09Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:37Z", - "generation": 1, - "labels": { - "accel-timeslicer": "true", - "app": "open-rl-trainer-worker", - "openrl.io/claim": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", - "openrl.io/role": "trainer", - "openrl.io/worker": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/group": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", - "timeslice.io/job-id": "lora-qwen-qwen3-8b-0-trainer", - "timeslice.io/owner": "qwen-qwen3-8b", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "openrl.io/v1alpha1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "Workload", - "name": "lora-qwen-qwen3-8b-0-trainer", - "uid": "d1f0b234-5386-41b2-b784-e32434f43d0b" - } - ], - "resourceVersion": "1792041342220143006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "In", - "values": [ - "true" - ] - } - ] - }, - { - "matchExpressions": [ - { - "key": "openrl.io/enabled", - "operator": "In", - "values": [ - "true" - ] - }, - { - "key": "openrl.io/trainer", - "operator": "DoesNotExist" - }, - { - "key": "openrl.io/sampler", - "operator": "DoesNotExist" - } - ] - } - ] - } - } - }, - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.hostIP" - } - } - }, - { - "name": "OPEN_RL_TIME_SLICE_GROUP", - "value": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234" - }, - { - "name": "OPEN_RL_CLAIM_LEDGER", - "value": "ledger-lora-qwen-qwen3-8b-0-trainer-d1f0b234" - }, - { - "name": "OPEN_RL_ASSIGNMENT_ID", - "value": "ea5c40aa-ef66-4cb9-ada2-49e019e38b8f" - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imagePullPolicy": "IfNotPresent", - "name": "worker", - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-t494m", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "resourceClaims": [ - { - "name": "gpu", - "resourceClaimName": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234" - } - ], - "restartPolicy": "OnFailure", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - }, - { - "name": "kube-api-access-t494m", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:15:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:15:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:15:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:15:42Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-15T05:15:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "memory": "36Gi" - }, - "containerID": "containerd://cebaa8843ee5dc2b32cec11699295319c90e89cd67412af9420e375cdf94cfb7", - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "imageID": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801", - "lastState": {}, - "name": "worker", - "ready": true, - "resources": { - "claims": [ - { - "name": "gpu" - } - ], - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-15T05:15:41Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-t494m", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.4.12", - "podIPs": [ - { - "ip": "10.32.4.12" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-15T05:15:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T01:28:42Z", - "generateName": "redis-store-86779df594-", - "generation": 1, - "labels": { - "app": "redis", - "pod-template-hash": "86779df594", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "redis-store-86779df594-7dc5x", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "redis-store-86779df594", - "uid": "74db3302-6e72-4c96-8fb7-7de729fc86f9" - } - ], - "resourceVersion": "1791941333868287008", - "uid": "e39bafca-6a3f-41a6-9942-64c1a6fb5744" - }, - "spec": { - "containers": [ - { - "image": "redis:7-alpine", - "imagePullPolicy": "IfNotPresent", - "name": "redis", - "ports": [ - { - "containerPort": 6379, - "protocol": "TCP" - } - ], - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "gke-open-rl-dra-default-pool-ec729728-6mpx", - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "default", - "serviceAccountName": "default", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "kube-api-access-fxgk2", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:53Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T01:28:42Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "allocatedResources": { - "cpu": "1", - "memory": "4Gi" - }, - "containerID": "containerd://cd152f5cfd3aa37e2660832d23c89f90b873fb29f3d1c0464e19acac23935bbf", - "image": "docker.io/library/redis:7-alpine", - "imageID": "docker.io/library/redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499", - "lastState": {}, - "name": "redis", - "ready": true, - "resources": { - "limits": { - "memory": "4Gi" - }, - "requests": { - "cpu": "1", - "memory": "4Gi" - } - }, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T01:28:52Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0, - 1, - 2, - 3, - 4, - 6, - 10, - 11, - 20, - 26, - 27 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-fxgk2", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.207", - "hostIPs": [ - { - "ip": "10.128.15.207" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.32.12.9", - "podIPs": [ - { - "ip": "10.32.12.9" - } - ], - "qosClass": "Burstable", - "startTime": "2026-10-14T01:28:42Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:05:36Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-9pkbv", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887950213247008", - "uid": "b1498158-ab9f-4a6a-91fd-2c80be598a94" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-mm47f", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:50Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:05:37Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://ca02ee4696d8e513f3817262e87899a1c277dbcc441912270216af53e1582f96", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:05:50Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-mm47f", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.201", - "hostIPs": [ - { - "ip": "10.128.15.201" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.201", - "podIPs": [ - { - "ip": "10.128.15.201" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:05:37Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:18:38Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-9wd9k", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944324015471012", - "uid": "ce71cee9-99bc-4214-bc1c-d522556068a2" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-tuqu" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-tuqu", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-5pb8r", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:43Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:18:38Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://e2c3f518f9671fd9786f2879537008ae20e493ccbc432c9e507f808c73dde172", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:18:43Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-5pb8r", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.216", - "hostIPs": [ - { - "ip": "10.128.15.216" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.216", - "podIPs": [ - { - "ip": "10.128.15.216" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:18:38Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:22:33Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-b5pg4", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789888987151695021", - "uid": "a8b69520-65ea-4b1e-bc15-77213a279a60" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pgqxm", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:23:07Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:22:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://03349a3992ebcdd981463c61332d3245437fed1514ec22320c3972098ce1a57f", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:23:06Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pgqxm", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.202", - "hostIPs": [ - { - "ip": "10.128.15.202" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.202", - "podIPs": [ - { - "ip": "10.128.15.202" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:22:33Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:03:24Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-dclxl", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789887814636575001", - "uid": "8484edfa-0dc8-4fac-a17a-36ce78fb8710" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-tq64h", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:34Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:03:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://dce9f7fa4663eb33d5815df9dea677b1b52c18d81054c1199da370f1c970d3a1", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:03:34Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-tq64h", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.196", - "hostIPs": [ - { - "ip": "10.128.15.196" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.196", - "podIPs": [ - { - "ip": "10.128.15.196" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:03:24Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-05T18:38:27Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-hcb9v", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791225513211215020", - "uid": "15b054e4-c82f-4cee-bf19-c0bfdcd8b577" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-0853" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-pbcm9", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:33Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-05T18:38:27Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://a8760998fb8f6c64ebcac928fb2af8964b56d3bb885e8b3eb952742d7435b181", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-05T18:38:32Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-pbcm9", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.197", - "hostIPs": [ - { - "ip": "10.128.15.197" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.197", - "podIPs": [ - { - "ip": "10.128.15.197" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-05T18:38:27Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-20T07:24:14Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-b" - }, - "name": "snapshot-agent-k8qtd", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1789889066585295015", - "uid": "bfa4c0fd-e19e-4be6-9d1c-6f96bef1c82a" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-bkd68", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:26Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-20T07:24:14Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://8b934ecb0fae9981de1fe2f716a013d5087d23c5b680e09822db36102b76b09d", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-20T07:24:25Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-bkd68", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.194", - "hostIPs": [ - { - "ip": "10.128.15.194" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.194", - "podIPs": [ - { - "ip": "10.128.15.194" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-20T07:24:14Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-09-30T18:57:25Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-lgbqp", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1790794650142511011", - "uid": "8e0ef9ca-5b6e-4ce8-a59e-2bc2167a9904" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-sampler-a721f127-ed3t" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-ed3t", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-966cp", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:30Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-09-30T18:57:25Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://cd5642bcb66630016bb1b35e619736d41de897d5f29607213d8f4d4150419895", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-09-30T18:57:29Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-966cp", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.0.10", - "hostIPs": [ - { - "ip": "10.128.0.10" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.0.10", - "podIPs": [ - { - "ip": "10.128.0.10" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-09-30T18:57:25Z" - } - }, - { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "creationTimestamp": "2026-10-14T02:19:20Z", - "generateName": "snapshot-agent-", - "generation": 1, - "labels": { - "app.kubernetes.io/instance": "snapshot-agent", - "app.kubernetes.io/name": "snapshot-agent", - "controller-revision-hash": "687c7f8b9f", - "pod-template-generation": "1", - "topology.kubernetes.io/region": "us-central1", - "topology.kubernetes.io/zone": "us-central1-a" - }, - "name": "snapshot-agent-vt9ct", - "namespace": "openrl-system", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "DaemonSet", - "name": "snapshot-agent", - "uid": "852816d4-070d-4086-8e5d-97cfe9b1e79e" - } - ], - "resourceVersion": "1791944364158111002", - "uid": "13db9aa6-0ca5-4e1b-bf3e-a802caab3be0" - }, - "spec": { - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchFields": [ - { - "key": "metadata.name", - "operator": "In", - "values": [ - "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - ] - } - ] - } - ] - } - } - }, - "containers": [ - { - "env": [ - { - "name": "DEPLOYMENT_MODE", - "value": "k8s" - }, - { - "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "spec.nodeName" - } - } - }, - { - "name": "AGENT_PORT", - "value": "9001" - }, - { - "name": "LD_LIBRARY_PATH", - "value": "/usr/local/nvidia/lib64" - }, - { - "name": "NVIDIA_VISIBLE_DEVICES", - "value": "all" - }, - { - "name": "NVIDIA_DRIVER_CAPABILITIES", - "value": "compute,utility" - } - ], - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imagePullPolicy": "Always", - "name": "snapshot-agent", - "ports": [ - { - "containerPort": 9001, - "hostPort": 9001, - "name": "grpc", - "protocol": "TCP" - } - ], - "resources": {}, - "securityContext": { - "privileged": true - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "hostNetwork": true, - "hostPID": true, - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "nodeSelector": { - "nvidia.com/gpu.present": "true" - }, - "preemptionPolicy": "PreemptLowerPriority", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "snapshot-agent", - "serviceAccountName": "snapshot-agent", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists" - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/disk-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/pid-pressure", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/unschedulable", - "operator": "Exists" - }, - { - "effect": "NoSchedule", - "key": "node.kubernetes.io/network-unavailable", - "operator": "Exists" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/home/kubernetes/bin/nvidia", - "type": "Directory" - }, - "name": "nvidia-driver" - }, - { - "hostPath": { - "path": "/dev", - "type": "Directory" - }, - "name": "nvidia-devices" - }, - { - "name": "kube-api-access-6vrn7", - "projected": { - "defaultMode": 420, - "sources": [ - { - "serviceAccountToken": { - "expirationSeconds": 3607, - "path": "token" - } - }, - { - "configMap": { - "items": [ - { - "key": "ca.crt", - "path": "ca.crt" - } - ], - "name": "kube-root-ca.crt" - } - }, - { - "downwardAPI": { - "items": [ - { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - }, - "path": "namespace" - } - ] - } - } - ] - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "PodReadyToStartContainers" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:24Z", - "observedGeneration": 1, - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2026-10-14T02:19:20Z", - "observedGeneration": 1, - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "containerd://7fa2a0e83934db24f10c55b5dda1e9636dc1a3878534f1920a80734d5f23bb82", - "image": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent:v0.1.0", - "imageID": "ghcr.io/llm-d-incubation/llm-d-rl-time-slicing/snapshot-agent@sha256:84fc38b10554c5a64f7e52f8bfd98f386406a807d1647a33502e0b8a2a4cf829", - "lastState": {}, - "name": "snapshot-agent", - "ready": true, - "resources": {}, - "restartCount": 0, - "started": true, - "state": { - "running": { - "startedAt": "2026-10-14T02:19:23Z" - } - }, - "user": { - "linux": { - "gid": 0, - "supplementalGroups": [ - 0 - ], - "uid": 0 - } - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia-driver", - "readOnly": true, - "recursiveReadOnly": "Disabled" - }, - { - "mountPath": "/dev", - "name": "nvidia-devices" - }, - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "kube-api-access-6vrn7", - "readOnly": true, - "recursiveReadOnly": "Disabled" - } - ] - } - ], - "hostIP": "10.128.15.211", - "hostIPs": [ - { - "ip": "10.128.15.211" - } - ], - "observedGeneration": 1, - "phase": "Running", - "podIP": "10.128.15.211", - "podIPs": [ - { - "ip": "10.128.15.211" - } - ], - "qosClass": "BestEffort", - "startTime": "2026-10-14T02:19:20Z" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log deleted file mode 100644 index 5d1efc70..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/redis-ready.log +++ /dev/null @@ -1,334 +0,0 @@ -2026-10-15T05:15:34.793+00:00 2026-10-15T05:15:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:34.794+00:00 2026-10-15T05:15:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:35.797+00:00 2026-10-15T05:15:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:35.797+00:00 2026-10-15T05:15:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:36.803+00:00 2026-10-15T05:15:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:36.803+00:00 2026-10-15T05:15:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:37.819+00:00 2026-10-15T05:15:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:37.819+00:00 2026-10-15T05:15:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:38.813+00:00 2026-10-15T05:15:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:38.814+00:00 2026-10-15T05:15:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:39.820+00:00 2026-10-15T05:15:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:39.820+00:00 2026-10-15T05:15:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:40.824+00:00 2026-10-15T05:15:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:40.824+00:00 2026-10-15T05:15:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:41.830+00:00 2026-10-15T05:15:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:41.830+00:00 2026-10-15T05:15:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:42.835+00:00 2026-10-15T05:15:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:42.835+00:00 2026-10-15T05:15:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:43.841+00:00 2026-10-15T05:15:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:43.841+00:00 2026-10-15T05:15:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:44.847+00:00 2026-10-15T05:15:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:44.847+00:00 2026-10-15T05:15:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:45.851+00:00 2026-10-15T05:15:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:45.851+00:00 2026-10-15T05:15:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:46.857+00:00 2026-10-15T05:15:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:46.857+00:00 2026-10-15T05:15:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:47.863+00:00 2026-10-15T05:15:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:47.863+00:00 2026-10-15T05:15:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:48.870+00:00 2026-10-15T05:15:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:48.870+00:00 2026-10-15T05:15:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:49.873+00:00 2026-10-15T05:15:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:49.873+00:00 2026-10-15T05:15:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:50.878+00:00 2026-10-15T05:15:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:50.878+00:00 2026-10-15T05:15:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:51.888+00:00 2026-10-15T05:15:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:51.889+00:00 2026-10-15T05:15:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:52.891+00:00 2026-10-15T05:15:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:52.892+00:00 2026-10-15T05:15:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:53.895+00:00 2026-10-15T05:15:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:53.895+00:00 2026-10-15T05:15:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:54.899+00:00 2026-10-15T05:15:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:54.899+00:00 2026-10-15T05:15:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:55.904+00:00 2026-10-15T05:15:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:55.904+00:00 2026-10-15T05:15:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:56.912+00:00 2026-10-15T05:15:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:56.912+00:00 2026-10-15T05:15:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:57.938+00:00 2026-10-15T05:15:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:57.938+00:00 2026-10-15T05:15:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:58.936+00:00 2026-10-15T05:15:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:58.936+00:00 2026-10-15T05:15:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:15:59.927+00:00 2026-10-15T05:15:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:15:59.927+00:00 2026-10-15T05:15:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:00.932+00:00 2026-10-15T05:16:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:00.932+00:00 2026-10-15T05:16:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:01.938+00:00 2026-10-15T05:16:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:01.938+00:00 2026-10-15T05:16:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:02.969+00:00 2026-10-15T05:16:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:02.970+00:00 2026-10-15T05:16:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:03.955+00:00 2026-10-15T05:16:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:03.955+00:00 2026-10-15T05:16:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:05.003+00:00 2026-10-15T05:16:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:05.004+00:00 2026-10-15T05:16:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:05.994+00:00 2026-10-15T05:16:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:05.995+00:00 2026-10-15T05:16:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:06.967+00:00 2026-10-15T05:16:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:06.967+00:00 2026-10-15T05:16:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:07.972+00:00 2026-10-15T05:16:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:07.972+00:00 2026-10-15T05:16:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:08.996+00:00 2026-10-15T05:16:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:08.996+00:00 2026-10-15T05:16:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:10.045+00:00 2026-10-15T05:16:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:10.045+00:00 2026-10-15T05:16:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:11.068+00:00 2026-10-15T05:16:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:11.068+00:00 2026-10-15T05:16:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:11.996+00:00 2026-10-15T05:16:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:11.997+00:00 2026-10-15T05:16:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:12.999+00:00 2026-10-15T05:16:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:12.999+00:00 2026-10-15T05:16:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:14.027+00:00 2026-10-15T05:16:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:14.028+00:00 2026-10-15T05:16:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:15.010+00:00 2026-10-15T05:16:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:15.011+00:00 2026-10-15T05:16:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:16.015+00:00 2026-10-15T05:16:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:16.015+00:00 2026-10-15T05:16:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:17.056+00:00 2026-10-15T05:16:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:17.057+00:00 2026-10-15T05:16:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:18.059+00:00 2026-10-15T05:16:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:18.059+00:00 2026-10-15T05:16:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:19.053+00:00 2026-10-15T05:16:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:19.053+00:00 2026-10-15T05:16:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:20.049+00:00 2026-10-15T05:16:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:20.049+00:00 2026-10-15T05:16:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:21.039+00:00 2026-10-15T05:16:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:21.039+00:00 2026-10-15T05:16:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:22.091+00:00 2026-10-15T05:16:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:22.091+00:00 2026-10-15T05:16:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:23.051+00:00 2026-10-15T05:16:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:23.051+00:00 2026-10-15T05:16:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:24.088+00:00 2026-10-15T05:16:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:24.088+00:00 2026-10-15T05:16:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:25.099+00:00 2026-10-15T05:16:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:25.099+00:00 2026-10-15T05:16:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:26.187+00:00 2026-10-15T05:16:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:26.187+00:00 2026-10-15T05:16:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:27.075+00:00 2026-10-15T05:16:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:27.075+00:00 2026-10-15T05:16:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:28.085+00:00 2026-10-15T05:16:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:28.086+00:00 2026-10-15T05:16:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:29.081+00:00 2026-10-15T05:16:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:29.081+00:00 2026-10-15T05:16:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:30.087+00:00 2026-10-15T05:16:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:30.087+00:00 2026-10-15T05:16:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:31.114+00:00 2026-10-15T05:16:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:31.114+00:00 2026-10-15T05:16:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:32.099+00:00 2026-10-15T05:16:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:32.099+00:00 2026-10-15T05:16:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:33.104+00:00 2026-10-15T05:16:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:33.104+00:00 2026-10-15T05:16:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:34.107+00:00 2026-10-15T05:16:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:34.108+00:00 2026-10-15T05:16:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:35.113+00:00 2026-10-15T05:16:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:35.113+00:00 2026-10-15T05:16:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:36.118+00:00 2026-10-15T05:16:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:36.119+00:00 2026-10-15T05:16:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:37.124+00:00 2026-10-15T05:16:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:37.125+00:00 2026-10-15T05:16:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:38.129+00:00 2026-10-15T05:16:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:38.129+00:00 2026-10-15T05:16:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:39.141+00:00 2026-10-15T05:16:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:39.141+00:00 2026-10-15T05:16:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:40.260+00:00 2026-10-15T05:16:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:40.260+00:00 2026-10-15T05:16:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:41.144+00:00 2026-10-15T05:16:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:41.144+00:00 2026-10-15T05:16:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:42.282+00:00 2026-10-15T05:16:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:42.282+00:00 2026-10-15T05:16:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:43.154+00:00 2026-10-15T05:16:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:43.154+00:00 2026-10-15T05:16:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:44.159+00:00 2026-10-15T05:16:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:44.159+00:00 2026-10-15T05:16:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:45.163+00:00 2026-10-15T05:16:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:45.163+00:00 2026-10-15T05:16:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:46.169+00:00 2026-10-15T05:16:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:46.169+00:00 2026-10-15T05:16:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:47.174+00:00 2026-10-15T05:16:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:47.175+00:00 2026-10-15T05:16:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:48.178+00:00 2026-10-15T05:16:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:48.178+00:00 2026-10-15T05:16:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:49.185+00:00 2026-10-15T05:16:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:49.185+00:00 2026-10-15T05:16:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:50.189+00:00 2026-10-15T05:16:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:50.189+00:00 2026-10-15T05:16:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:51.194+00:00 2026-10-15T05:16:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:51.195+00:00 2026-10-15T05:16:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:52.199+00:00 2026-10-15T05:16:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:52.199+00:00 2026-10-15T05:16:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:53.225+00:00 2026-10-15T05:16:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:53.225+00:00 2026-10-15T05:16:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:54.208+00:00 2026-10-15T05:16:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:54.208+00:00 2026-10-15T05:16:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:55.215+00:00 2026-10-15T05:16:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:55.215+00:00 2026-10-15T05:16:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:56.219+00:00 2026-10-15T05:16:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:56.219+00:00 2026-10-15T05:16:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:57.224+00:00 2026-10-15T05:16:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:57.224+00:00 2026-10-15T05:16:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:58.229+00:00 2026-10-15T05:16:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:58.229+00:00 2026-10-15T05:16:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:16:59.235+00:00 2026-10-15T05:16:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:16:59.235+00:00 2026-10-15T05:16:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:00.240+00:00 2026-10-15T05:17:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:00.240+00:00 2026-10-15T05:17:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:01.244+00:00 2026-10-15T05:17:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:01.245+00:00 2026-10-15T05:17:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:02.248+00:00 2026-10-15T05:17:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:02.249+00:00 2026-10-15T05:17:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:03.255+00:00 2026-10-15T05:17:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:03.255+00:00 2026-10-15T05:17:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:04.261+00:00 2026-10-15T05:17:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:04.261+00:00 2026-10-15T05:17:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:05.266+00:00 2026-10-15T05:17:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:05.266+00:00 2026-10-15T05:17:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:06.271+00:00 2026-10-15T05:17:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:06.272+00:00 2026-10-15T05:17:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:07.278+00:00 2026-10-15T05:17:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:07.278+00:00 2026-10-15T05:17:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:08.283+00:00 2026-10-15T05:17:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:08.283+00:00 2026-10-15T05:17:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:09.291+00:00 2026-10-15T05:17:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:09.291+00:00 2026-10-15T05:17:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:10.294+00:00 2026-10-15T05:17:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:10.294+00:00 2026-10-15T05:17:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:11.299+00:00 2026-10-15T05:17:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:11.299+00:00 2026-10-15T05:17:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:12.304+00:00 2026-10-15T05:17:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:12.304+00:00 2026-10-15T05:17:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:13.309+00:00 2026-10-15T05:17:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:13.309+00:00 2026-10-15T05:17:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:14.314+00:00 2026-10-15T05:17:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:14.314+00:00 2026-10-15T05:17:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:15.499+00:00 2026-10-15T05:17:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:15.499+00:00 2026-10-15T05:17:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:16.325+00:00 2026-10-15T05:17:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:16.325+00:00 2026-10-15T05:17:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:17.329+00:00 2026-10-15T05:17:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:17.330+00:00 2026-10-15T05:17:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:18.344+00:00 2026-10-15T05:17:18Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:18.344+00:00 2026-10-15T05:17:18Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:19.340+00:00 2026-10-15T05:17:19Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:19.340+00:00 2026-10-15T05:17:19Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:20.345+00:00 2026-10-15T05:17:20Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:20.345+00:00 2026-10-15T05:17:20Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:21.349+00:00 2026-10-15T05:17:21Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:21.349+00:00 2026-10-15T05:17:21Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:22.363+00:00 2026-10-15T05:17:22Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:22.364+00:00 2026-10-15T05:17:22Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:23.362+00:00 2026-10-15T05:17:23Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:23.362+00:00 2026-10-15T05:17:23Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:24.365+00:00 2026-10-15T05:17:24Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:24.365+00:00 2026-10-15T05:17:24Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:25.371+00:00 2026-10-15T05:17:25Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:25.371+00:00 2026-10-15T05:17:25Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:26.375+00:00 2026-10-15T05:17:26Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:26.375+00:00 2026-10-15T05:17:26Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:27.381+00:00 2026-10-15T05:17:27Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:27.381+00:00 2026-10-15T05:17:27Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:28.386+00:00 2026-10-15T05:17:28Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:28.386+00:00 2026-10-15T05:17:28Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:29.390+00:00 2026-10-15T05:17:29Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:29.391+00:00 2026-10-15T05:17:29Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:30.398+00:00 2026-10-15T05:17:30Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:30.398+00:00 2026-10-15T05:17:30Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:31.433+00:00 2026-10-15T05:17:31Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:31.434+00:00 2026-10-15T05:17:31Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:32.409+00:00 2026-10-15T05:17:32Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:32.409+00:00 2026-10-15T05:17:32Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:33.414+00:00 2026-10-15T05:17:33Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:33.414+00:00 2026-10-15T05:17:33Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:34.418+00:00 2026-10-15T05:17:34Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:34.419+00:00 2026-10-15T05:17:34Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:35.424+00:00 2026-10-15T05:17:35Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:35.424+00:00 2026-10-15T05:17:35Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:36.429+00:00 2026-10-15T05:17:36Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:36.429+00:00 2026-10-15T05:17:36Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:37.435+00:00 2026-10-15T05:17:37Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:37.436+00:00 2026-10-15T05:17:37Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:38.463+00:00 2026-10-15T05:17:38Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:38.463+00:00 2026-10-15T05:17:38Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:39.445+00:00 2026-10-15T05:17:39Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:39.445+00:00 2026-10-15T05:17:39Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:40.449+00:00 2026-10-15T05:17:40Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:40.449+00:00 2026-10-15T05:17:40Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:41.454+00:00 2026-10-15T05:17:41Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:41.454+00:00 2026-10-15T05:17:41Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:42.469+00:00 2026-10-15T05:17:42Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:42.469+00:00 2026-10-15T05:17:42Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:43.464+00:00 2026-10-15T05:17:43Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:43.465+00:00 2026-10-15T05:17:43Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:44.469+00:00 2026-10-15T05:17:44Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:44.469+00:00 2026-10-15T05:17:44Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:45.475+00:00 2026-10-15T05:17:45Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:45.475+00:00 2026-10-15T05:17:45Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:46.632+00:00 2026-10-15T05:17:46Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:46.632+00:00 2026-10-15T05:17:46Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:47.485+00:00 2026-10-15T05:17:47Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:47.485+00:00 2026-10-15T05:17:47Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:48.489+00:00 2026-10-15T05:17:48Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:48.490+00:00 2026-10-15T05:17:48Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:49.495+00:00 2026-10-15T05:17:49Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:49.495+00:00 2026-10-15T05:17:49Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:50.501+00:00 2026-10-15T05:17:50Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:50.501+00:00 2026-10-15T05:17:50Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:51.508+00:00 2026-10-15T05:17:51Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:51.508+00:00 2026-10-15T05:17:51Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:52.511+00:00 2026-10-15T05:17:52Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:52.511+00:00 2026-10-15T05:17:52Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:53.516+00:00 2026-10-15T05:17:53Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:53.517+00:00 2026-10-15T05:17:53Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:54.521+00:00 2026-10-15T05:17:54Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:54.521+00:00 2026-10-15T05:17:54Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:55.527+00:00 2026-10-15T05:17:55Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:55.527+00:00 2026-10-15T05:17:55Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:56.530+00:00 2026-10-15T05:17:56Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:56.530+00:00 2026-10-15T05:17:56Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:57.538+00:00 2026-10-15T05:17:57Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:57.538+00:00 2026-10-15T05:17:57Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:58.541+00:00 2026-10-15T05:17:58Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:58.541+00:00 2026-10-15T05:17:58Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:17:59.546+00:00 2026-10-15T05:17:59Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:17:59.546+00:00 2026-10-15T05:17:59Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:00.551+00:00 2026-10-15T05:18:00Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:00.552+00:00 2026-10-15T05:18:00Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:01.567+00:00 2026-10-15T05:18:01Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:01.567+00:00 2026-10-15T05:18:01Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:02.561+00:00 2026-10-15T05:18:02Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:02.561+00:00 2026-10-15T05:18:02Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:03.566+00:00 2026-10-15T05:18:03Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:03.566+00:00 2026-10-15T05:18:03Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:04.573+00:00 2026-10-15T05:18:04Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:04.573+00:00 2026-10-15T05:18:04Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:05.577+00:00 2026-10-15T05:18:05Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:05.577+00:00 2026-10-15T05:18:05Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:06.582+00:00 2026-10-15T05:18:06Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:06.582+00:00 2026-10-15T05:18:06Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:07.589+00:00 2026-10-15T05:18:07Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:07.589+00:00 2026-10-15T05:18:07Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:08.594+00:00 2026-10-15T05:18:08Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:08.595+00:00 2026-10-15T05:18:08Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:09.599+00:00 2026-10-15T05:18:09Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:09.599+00:00 2026-10-15T05:18:09Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:10.606+00:00 2026-10-15T05:18:10Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:10.606+00:00 2026-10-15T05:18:10Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:10.606+00:00 2026-10-15T05:18:10Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:11.610+00:00 2026-10-15T05:18:11Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:11.610+00:00 2026-10-15T05:18:11Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:11.610+00:00 2026-10-15T05:18:11Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:12.616+00:00 2026-10-15T05:18:12Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:12.616+00:00 2026-10-15T05:18:12Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:12.616+00:00 2026-10-15T05:18:12Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:13.638+00:00 2026-10-15T05:18:13Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:13.638+00:00 2026-10-15T05:18:13Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:13.638+00:00 2026-10-15T05:18:13Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:14.627+00:00 2026-10-15T05:18:14Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:14.628+00:00 2026-10-15T05:18:14Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:14.628+00:00 2026-10-15T05:18:14Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:15.631+00:00 2026-10-15T05:18:15Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:15.631+00:00 2026-10-15T05:18:15Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:15.631+00:00 2026-10-15T05:18:15Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:16.637+00:00 2026-10-15T05:18:16Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:16.637+00:00 2026-10-15T05:18:16Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:16.637+00:00 2026-10-15T05:18:16Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B -2026-10-15T05:18:17.642+00:00 2026-10-15T05:18:17Z open_rl:sampler_ready:Qwen/Qwen3-8B -2026-10-15T05:18:17.642+00:00 2026-10-15T05:18:17Z open_rl:trainer_ready:Qwen/Qwen3-8B -2026-10-15T05:18:17.642+00:00 2026-10-15T05:18:17Z open_rl:trainer_ready:Qwen/Qwen2.5-0.5B diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log deleted file mode 100644 index 67f925c9..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/scheduler.log +++ /dev/null @@ -1,2 +0,0 @@ -2026-10-15T05:15:37.772+00:00 2026-10-15T05:15:37.770103723Z {"level":"info","ts":"2026-10-15T05:15:37Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-trainer","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-trainer","reconcileID":"6e1c65ba-9430-4398-89db-8b05b9f2092e","claim":"claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234","worker":"lora-qwen-qwen3-8b-0-trainer","tiers":"t1x80"} -2026-10-15T05:16:09.217+00:00 2026-10-15T05:16:09.211255887Z {"level":"info","ts":"2026-10-15T05:16:09Z","msg":"cutting a claim","controller":"workload","controllerGroup":"openrl.io","controllerKind":"Workload","Workload":{"name":"lora-qwen-qwen3-8b-0-sampler","namespace":"openrl-system"},"namespace":"openrl-system","name":"lora-qwen-qwen3-8b-0-sampler","reconcileID":"d4df84da-7e1d-497e-abe9-39f53b544303","claim":"claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6","worker":"lora-qwen-qwen3-8b-0-sampler","tiers":"t1x80"} diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json deleted file mode 100644 index 4035e7bf..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/timeline.json +++ /dev/null @@ -1,267 +0,0 @@ -{ - "case": "after-D2-8b-h100-cached", - "collected": "2026-10-15T05:18:18.329+00:00", - "client": { - "client.start": { - "ts": "2026-10-15 05:15:36.633000+00:00", - "source": "client.jsonl (laptop clock)", - "base_model": "Qwen/Qwen3-8B", - "base_url": "http://127.0.0.1:9003" - }, - "client.create_model.start": { - "ts": "2026-10-15 05:15:36.633000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.create_model.done": { - "ts": "2026-10-15 05:16:00.765000+00:00", - "source": "client.jsonl (laptop clock)", - "model_id": "ce0e7790-7ff6-44f1-9632-06939e7b1053" - }, - "client.forward_backward.start": { - "ts": "2026-10-15 05:16:03.882000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.forward_backward.done": { - "ts": "2026-10-15 05:16:06.825000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.optim_step.done": { - "ts": "2026-10-15 05:16:08.923000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.start": { - "ts": "2026-10-15 05:16:08.924000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.save_weights_and_get_sampling_client.done": { - "ts": "2026-10-15 05:16:11.071000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.start": { - "ts": "2026-10-15 05:16:11.076000+00:00", - "source": "client.jsonl (laptop clock)" - }, - "client.sample.done": { - "ts": "2026-10-15 05:18:10.965000+00:00", - "source": "client.jsonl (laptop clock)", - "text": " 4\n\nQuestion: What is " - }, - "client.end": { - "ts": "2026-10-15 05:18:10.966000+00:00", - "source": "client.jsonl (laptop clock)" - } - }, - "workers": { - "sampler": { - "workload": "lora-qwen-qwen3-8b-0-sampler", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "pod": "orw-lora-qwen-qwen3-8b-0-sampler", - "marks": { - "sampler.workload_created": { - "ts": "2026-10-15 05:16:09+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "sampler.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:16:09.599000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "sampler.workload_reports_pod": { - "ts": "2026-10-15 05:16:09.599000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "sampler.workload_reports_node": { - "ts": "2026-10-15 05:16:09.599000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "sampler.condition_Placed_True": { - "ts": "2026-10-15 05:16:13+00:00", - "source": "Workload condition Placed" - }, - "sampler.pod_created": { - "ts": "2026-10-15 05:16:09+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "sampler.pod_Initialized": { - "ts": "2026-10-15 05:16:09+00:00", - "source": "pod condition Initialized" - }, - "sampler.pod_Ready": { - "ts": "2026-10-15 05:16:13+00:00", - "source": "pod condition Ready" - }, - "sampler.pod_ContainersReady": { - "ts": "2026-10-15 05:16:13+00:00", - "source": "pod condition ContainersReady" - }, - "sampler.pod_PodScheduled": { - "ts": "2026-10-15 05:16:09+00:00", - "source": "pod condition PodScheduled" - }, - "sampler.container_started": { - "ts": "2026-10-15 05:16:13+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "sampler.event_Scheduled": { - "ts": "2026-10-15 05:16:09+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "sampler.event_Pulled": { - "ts": "2026-10-15 05:16:13+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "sampler.event_Created": { - "ts": "2026-10-15 05:16:13+00:00", - "source": "event: Container created" - }, - "sampler.event_Started": { - "ts": "2026-10-15 05:16:13+00:00", - "source": "event: Container started" - }, - "sampler.process_first_log": { - "ts": "2026-10-15 05:16:28.490961+00:00", - "source": "container log, first line" - }, - "sampler.engine_init_start": { - "ts": "2026-10-15 05:16:38.151495+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:38 [core.py:123] Initializing a V1 LLM engine (v0.30.0) with config: model='Qwen/Q" - }, - "sampler.load_start": { - "ts": "2026-10-15 05:16:39.244394+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:39 [model_runner.py:396] Loading model from scratch..." - }, - "sampler.prefetch_start": { - "ts": "2026-10-15 05:16:40.237765+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:40 [weight_utils.py:857] Prefetching checkpoint files into page cache started (in " - }, - "sampler.weights_loaded": { - "ts": "2026-10-15 05:16:52.391122+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:52 [default_loader.py:430] Loading weights took 12.16 seconds", - "self_reported_s": 12.16 - }, - "sampler.model_loaded": { - "ts": "2026-10-15 05:16:53.020747+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:16:53 [model_runner.py:428] Model loading took 18.17 GiB memory and 14.053276 seconds", - "self_reported_s": 14.053276 - }, - "sampler.compile_done": { - "ts": "2026-10-15 05:17:29.578400+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:17:29 [monitor.py:53] torch.compile took 35.52 s in total", - "self_reported_s": 35.52 - }, - "sampler.graphs_captured": { - "ts": "2026-10-15 05:17:49.866634+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:17:49 [model_runner.py:1066] Graph capturing finished in 16 secs, took 0.78 GiB", - "self_reported_s": 16.0 - }, - "sampler.kv_cache_sized": { - "ts": "2026-10-15 05:17:50.489301+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:17:50 [gpu_worker.py:640] Available KV cache memory: 50.17 GiB" - }, - "sampler.engine_init_done": { - "ts": "2026-10-15 05:18:09.053659+00:00", - "source": "container log: (EngineCore pid=169) INFO 10-15 05:18:09 [core.py:372] init engine (profile, create kv cache, warmup model) took 76.03 s", - "self_reported_s": 76.03 - }, - "sampler.ready_key": { - "ts": "2026-10-15 05:18:10+00:00", - "source": "redis poller: open_rl:sampler_ready:Qwen/Qwen3-8B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - }, - "trainer": { - "workload": "lora-qwen-qwen3-8b-0-trainer", - "model_id": "Qwen/Qwen3-8B", - "node": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "pod": "orw-lora-qwen-qwen3-8b-0-trainer", - "marks": { - "trainer.workload_created": { - "ts": "2026-10-15 05:15:37+00:00", - "source": "Workload creationTimestamp (apiserver)" - }, - "trainer.workload_first_seen_by_poller": { - "ts": "2026-10-15 05:15:37.332000+00:00", - "source": "workloads.tsv poller (laptop clock, 1 s)" - }, - "trainer.workload_reports_pod": { - "ts": "2026-10-15 05:15:37.332000+00:00", - "source": "workloads.tsv poller: status.podName set" - }, - "trainer.workload_reports_node": { - "ts": "2026-10-15 05:15:38.981000+00:00", - "source": "workloads.tsv poller: status.nodeName set" - }, - "trainer.condition_Placed_True": { - "ts": "2026-10-15 05:15:42+00:00", - "source": "Workload condition Placed" - }, - "trainer.pod_created": { - "ts": "2026-10-15 05:15:37+00:00", - "source": "pod creationTimestamp (apiserver)" - }, - "trainer.pod_Initialized": { - "ts": "2026-10-15 05:15:38+00:00", - "source": "pod condition Initialized" - }, - "trainer.pod_Ready": { - "ts": "2026-10-15 05:15:42+00:00", - "source": "pod condition Ready" - }, - "trainer.pod_ContainersReady": { - "ts": "2026-10-15 05:15:42+00:00", - "source": "pod condition ContainersReady" - }, - "trainer.pod_PodScheduled": { - "ts": "2026-10-15 05:15:37+00:00", - "source": "pod condition PodScheduled" - }, - "trainer.container_started": { - "ts": "2026-10-15 05:15:41+00:00", - "source": "containerStatuses.state.running.startedAt" - }, - "trainer.event_Scheduled": { - "ts": "2026-10-15 05:15:37+00:00", - "source": "event: Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "trainer.event_Pulled": { - "ts": "2026-10-15 05:15:41+00:00", - "source": "event: Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and ", - "kubelet_pull_s": null, - "pulled_from_cache": true - }, - "trainer.event_Created": { - "ts": "2026-10-15 05:15:41+00:00", - "source": "event: Container created" - }, - "trainer.event_Started": { - "ts": "2026-10-15 05:15:41+00:00", - "source": "event: Container started" - }, - "trainer.process_first_log": { - "ts": "2026-10-15 05:15:45.886784+00:00", - "source": "container log, first line" - }, - "trainer.load_start": { - "ts": "2026-10-15 05:15:46.108440+00:00", - "source": "container log: Loading base model Qwen/Qwen3-8B to cuda:0..." - }, - "trainer.load_done": { - "ts": "2026-10-15 05:15:58.258108+00:00", - "source": "container log: Successfully loaded." - }, - "trainer.processor_started": { - "ts": "2026-10-15 05:15:58.259517+00:00", - "source": "container log: [WORKER] Training requests processor started (model=Qwen/Qwen3-8B active_set=Qwen/Qwen3-8B-1)." - }, - "trainer.ready_key": { - "ts": "2026-10-15 05:15:34+00:00", - "source": "redis poller: open_rl:trainer_ready:Qwen/Qwen2.5-0.5B" - } - }, - "image_id": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server@sha256:2e1c95e42a4b43d72496e9a87cb6c299baf3568eea533d01c3bbd84c6bb03801" - } - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt deleted file mode 100644 index e83592ef..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/watch-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:15:32.369+00:00 diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt deleted file mode 100644 index a9917932..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/worker-image.txt +++ /dev/null @@ -1 +0,0 @@ -us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json b/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json deleted file mode 100644 index 9df2e608..00000000 --- a/runs/2026-10-15_cold_start/after-D2-8b-h100-cached/workloads.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:09Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041373802335008", - "uid": "0c46d0b6-726c-4452-a321-7e87b66cd63b" - }, - "spec": { - "accelerator": { - "memory": "29Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "sampler", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.vllm_sampler" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "30725539152" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "OPEN_RL_MODEL_ID", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "VLLM_SERVER_DEV_MODE", - "value": "1" - }, - { - "name": "VLLM_ALLOW_INSECURE_SERIALIZATION", - "value": "1" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-sampler" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "40Gi" - }, - "requests": { - "memory": "40Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "35cbcea6-3b99-4526-8bc3-c6a94bb35da6", - "claimName": "claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:16:13Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-sampler-0c46d0b6", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "29Gi", - "nodeName": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-sampler" - } - }, - { - "apiVersion": "openrl.io/v1alpha1", - "kind": "Workload", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:37Z", - "finalizers": [ - "openrl.io/placement" - ], - "generation": 2, - "labels": { - "app.kubernetes.io/managed-by": "open-rl-api-server" - }, - "name": "lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041342325231004", - "uid": "d1f0b234-5386-41b2-b784-e32434f43d0b" - }, - "spec": { - "accelerator": { - "memory": "20Gi", - "mode": "SingleGPU" - }, - "exclusive": true, - "modelID": "Qwen/Qwen3-8B", - "ownerID": "qwen-qwen3-8b", - "role": "trainer", - "template": { - "metadata": {}, - "spec": { - "containers": [ - { - "args": [ - "--model-id", - "Qwen/Qwen3-8B", - "--active-tenant-set-id", - "Qwen/Qwen3-8B-1" - ], - "command": [ - "uv", - "run", - "python", - "-u", - "-m", - "server.training_requests_processor" - ], - "env": [ - { - "name": "REDIS_URL", - "value": "redis://redis-service:6379?max_connections=500" - }, - { - "name": "OPEN_RL_TMP_DIR", - "value": "/mnt/shared/open-rl" - }, - { - "name": "HF_HOME", - "value": "/mnt/shared/open-rl/huggingface" - }, - { - "name": "BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_BASE_MODEL", - "value": "Qwen/Qwen3-8B" - }, - { - "name": "OPEN_RL_ENABLE_FFT", - "value": "false" - }, - { - "name": "OPEN_RL_FINE_TUNING_TYPE", - "value": "lora" - }, - { - "name": "OPEN_RL_ACCELERATOR_MEMORY", - "value": "20674967296" - }, - { - "name": "OPEN_RL_WEIGHT_SYNC_STRATEGY", - "value": "delta" - }, - { - "name": "PYTORCH_CUDA_ALLOC_CONF", - "value": "expandable_segments:True" - }, - { - "name": "OPEN_RL_WORKLOAD_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_TIME_SLICE_JOB_ID", - "value": "lora-qwen-qwen3-8b-0-trainer" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_PORT", - "value": "9753" - }, - { - "name": "OPEN_RL_TIME_SLICING", - "value": "off" - }, - { - "name": "OPEN_RL_ACCEL_TIMESLICER_HOST", - "valueFrom": { - "fieldRef": { - "fieldPath": "status.hostIP" - } - } - } - ], - "image": "us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a", - "name": "worker", - "resources": { - "limits": { - "memory": "36Gi" - }, - "requests": { - "memory": "36Gi" - } - }, - "volumeMounts": [ - { - "mountPath": "/mnt/shared", - "name": "shared-storage" - } - ] - } - ], - "restartPolicy": "OnFailure", - "tolerations": [ - { - "effect": "NoSchedule", - "key": "nvidia.com/gpu", - "operator": "Exists" - } - ], - "volumes": [ - { - "name": "shared-storage", - "persistentVolumeClaim": { - "claimName": "open-rl-shared-pvc" - } - } - ] - } - }, - "trainingKind": "lora", - "workerContainerName": "worker" - }, - "status": { - "assignmentID": "ea5c40aa-ef66-4cb9-ada2-49e019e38b8f", - "claimName": "claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", - "conditions": [ - { - "lastTransitionTime": "2026-10-15T05:15:42Z", - "message": "worker is running on claim-lora-qwen-qwen3-8b-0-trainer-d1f0b234", - "reason": "Placed", - "status": "True", - "type": "Placed" - } - ], - "deviceCount": 1, - "memoryPerDevice": "20Gi", - "nodeName": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "observedGeneration": 2, - "phase": "Running", - "podName": "orw-lora-qwen-qwen3-8b-0-trainer" - } - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log deleted file mode 100644 index fc418348..00000000 --- a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/api-server.log +++ /dev/null @@ -1,26 +0,0 @@ -2026-10-15T05:19:17.204+00:00 2026-10-15T05:19:17.207002187Z INFO: 127.0.0.1:58394 - "POST /api/v1/client/config HTTP/1.1" 200 OK -2026-10-15T05:19:17.439+00:00 2026-10-15T05:19:17.441717541Z INFO: 127.0.0.1:58408 - "POST /api/v1/client/dynamic_config HTTP/1.1" 404 Not Found -2026-10-15T05:19:17.507+00:00 2026-10-15T05:19:17.509980459Z INFO: 127.0.0.1:58408 - "POST /api/v1/create_session HTTP/1.1" 200 OK -2026-10-15T05:19:17.741+00:00 2026-10-15T05:19:17.744372268Z [API_SERVER] enqueued op=create_model request_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 model_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 active_set=google/gemma-4-e2b-1 -2026-10-15T05:19:17.742+00:00 2026-10-15T05:19:17.745324931Z INFO: 127.0.0.1:58412 - "POST /api/v1/create_model HTTP/1.1" 200 OK -2026-10-15T05:19:20.745+00:00 2026-10-15T05:19:20.746976080Z INFO: 10.32.3.1:37564 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:19:27.698+00:00 2026-10-15T05:19:27.698776198Z INFO: 127.0.0.1:52116 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:19:30.745+00:00 2026-10-15T05:19:30.747139576Z INFO: 10.32.3.1:37230 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:19:40.747+00:00 2026-10-15T05:19:40.747233817Z INFO: 10.32.3.1:48412 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:19:50.747+00:00 2026-10-15T05:19:50.746985238Z INFO: 10.32.3.1:41220 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:20:00.746+00:00 2026-10-15T05:20:00.747586306Z INFO: 10.32.3.1:59588 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:20:07.948+00:00 2026-10-15T05:20:07.949379797Z INFO: 127.0.0.1:54766 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:20:10.744+00:00 2026-10-15T05:20:10.747103480Z INFO: 10.32.3.1:40408 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:20:20.745+00:00 2026-10-15T05:20:20.746536607Z INFO: 10.32.3.1:43004 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:20:30.745+00:00 2026-10-15T05:20:30.746837230Z INFO: 10.32.3.1:59800 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:20:32.447+00:00 2026-10-15T05:20:32.450365504Z [API_SERVER] No live session uses qwen-qwen3-8b; tearing its workers down -2026-10-15T05:20:40.746+00:00 2026-10-15T05:20:40.746710674Z INFO: 10.32.3.1:58932 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:20:50.745+00:00 2026-10-15T05:20:50.746442915Z INFO: 10.32.3.1:54538 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:20:58.177+00:00 2026-10-15T05:20:58.178160356Z INFO: 127.0.0.1:50924 - "POST /api/v1/telemetry HTTP/1.1" 200 OK -2026-10-15T05:21:00.745+00:00 2026-10-15T05:21:00.746518377Z INFO: 10.32.3.1:48928 - "GET /api/v1/healthz HTTP/1.1" 200 OK -2026-10-15T05:21:00.838+00:00 2026-10-15T05:21:00.842106688Z INFO: 127.0.0.1:50936 - "POST /api/v1/get_info HTTP/1.1" 200 OK -2026-10-15T05:21:04.579+00:00 2026-10-15T05:21:04.560065189Z [API_SERVER] enqueued op=forward_backward request_id=78e99e90-36d1-4203-863a-1cc73a5c1ee1 model_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 active_set=google/gemma-4-e2b-1 -2026-10-15T05:21:04.580+00:00 2026-10-15T05:21:04.560605605Z INFO: 127.0.0.1:50936 - "POST /api/v1/forward_backward HTTP/1.1" 200 OK -2026-10-15T05:21:08.105+00:00 2026-10-15T05:21:08.106319850Z [API_SERVER] enqueued op=optim_step request_id=de504127-e705-4531-a2cc-f37ca7755809 model_id=c0c2205d-aa11-49f3-9bfe-4dafd98e0314 active_set=google/gemma-4-e2b-1 -2026-10-15T05:21:08.105+00:00 2026-10-15T05:21:08.107759850Z INFO: 127.0.0.1:50936 - "POST /api/v1/optim_step HTTP/1.1" 200 OK -2026-10-15T05:21:10.744+00:00 2026-10-15T05:21:10.747068885Z INFO: 10.32.3.1:48390 - "GET /api/v1/healthz HTTP/1.1" 200 OK diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl deleted file mode 100644 index 9a091937..00000000 --- a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.jsonl +++ /dev/null @@ -1,7 +0,0 @@ -{"ts": "2026-10-15T05:19:16.955+00:00", "t": 0.0, "phase": "start", "base_model": "google/gemma-4-e2b", "base_url": "http://127.0.0.1:9003"} -{"ts": "2026-10-15T05:19:16.956+00:00", "t": 0.001, "phase": "create_model.start"} -{"ts": "2026-10-15T05:21:00.658+00:00", "t": 103.703, "phase": "create_model.done", "model_id": "c0c2205d-aa11-49f3-9bfe-4dafd98e0314"} -{"ts": "2026-10-15T05:21:04.495+00:00", "t": 107.54, "phase": "forward_backward.start"} -{"ts": "2026-10-15T05:21:08.031+00:00", "t": 111.076, "phase": "forward_backward.done"} -{"ts": "2026-10-15T05:21:09.190+00:00", "t": 112.235, "phase": "optim_step.done"} -{"ts": "2026-10-15T05:21:09.196+00:00", "t": 112.241, "phase": "end"} diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log deleted file mode 100644 index 8fae29fe..00000000 --- a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/client.log +++ /dev/null @@ -1,8 +0,0 @@ -[cold-start] 0.000s start {'base_model': 'google/gemma-4-e2b', 'base_url': 'http://127.0.0.1:9003'} -[cold-start] 0.001s create_model.start -[cold-start] 103.703s create_model.done {'model_id': 'c0c2205d-aa11-49f3-9bfe-4dafd98e0314'} -[cold-start] 107.540s forward_backward.start -[cold-start] 111.076s forward_backward.done -[cold-start] 112.235s optim_step.done -[cold-start] 112.241s end -EXIT=0 diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt deleted file mode 100644 index a7b79641..00000000 --- a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/collect-started.txt +++ /dev/null @@ -1 +0,0 @@ -2026-10-15T05:21:09.752+00:00 diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json deleted file mode 100644 index 89e9f845..00000000 --- a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/events.json +++ /dev/null @@ -1,3265 +0,0 @@ -{ - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:10Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:10Z", - "name": "cold-start-fio.18de9a0ef295773d", - "namespace": "openrl-system", - "resourceVersion": "1792039090760383011", - "uid": "d5915a58-e6d2-4888-8c0e-e6150aea91fc" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 1.766s (1.766s including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5be63862", - "namespace": "openrl-system", - "resourceVersion": "1792039092515007000", - "uid": "355b59b9-3661-499d-8640-3f40c3c89375" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f5ecfa51a", - "namespace": "openrl-system", - "resourceVersion": "1792039092561663021", - "uid": "9e5f01c5-89cd-4473-b881-48a8a996fbda" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:38:12Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:38:12Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:38:12Z", - "name": "cold-start-fio.18de9a0f61ddd1e4", - "namespace": "openrl-system", - "resourceVersion": "1792039092605935011", - "uid": "d673685d-16ba-4d65-b17b-b425c1d2f6ab" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:40:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039086935807002", - "uid": "4def5a0a-832d-451c-a3ef-cab03d0d91d5" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:40:41Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:40:41Z", - "name": "cold-start-fio.18de9a32081cf908", - "namespace": "openrl-system", - "resourceVersion": "1792039241446287009", - "uid": "14a62e42-13ba-4526-a04d-a934702e1163" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:18Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:18Z", - "message": "Pulling image \"ubuntu:24.04\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:18Z", - "name": "cold-start-fio.18de9a3a9867ce9f", - "namespace": "openrl-system", - "resourceVersion": "1792039278208207001", - "uid": "974022d7-16ec-4c0d-bf3f-7203aef9e6b6" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Successfully pulled image \"ubuntu:24.04\" in 978ms (978ms including waiting). Image size: 29773586 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad2bcac91", - "namespace": "openrl-system", - "resourceVersion": "1792039279188751014", - "uid": "ed3533ff-4205-4005-b7dd-d6994fe186bd" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad378cd74", - "namespace": "openrl-system", - "resourceVersion": "1792039279219679006", - "uid": "deffd093-c0a5-497c-8d57-72dd515dff4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:41:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:41:19Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:41:19Z", - "name": "cold-start-fio.18de9a3ad5aed221", - "namespace": "openrl-system", - "resourceVersion": "1792039279235135013", - "uid": "e37f2e68-3a75-4642-ad00-51c33c69cf40" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:11Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{fio}", - "kind": "Pod", - "name": "cold-start-fio", - "namespace": "openrl-system", - "resourceVersion": "1792039274774991002", - "uid": "06e55e33-14ac-474f-8d1e-229ef51ea557" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:11Z", - "message": "Stopping container fio", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:11Z", - "name": "cold-start-fio.18de9a70cd2c6a68", - "namespace": "openrl-system", - "resourceVersion": "1792039511048479001", - "uid": "e380f162-cff4-4e8b-9f74-c76902b75e47" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-645ff9777d-zg9cl", - "namespace": "openrl-system", - "resourceVersion": "1791941114537823021", - "uid": "60157023-ba75-4943-bf11-582e2b35f2e6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Stopping container api-server", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d-zg9cl.18de9a78ce247435", - "namespace": "openrl-system", - "resourceVersion": "1792039545418735013", - "uid": "ea8815fe-134f-4492-ac25-d9e75ec8d9d6" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-645ff9777d", - "namespace": "openrl-system", - "resourceVersion": "1792039545300351007", - "uid": "944f738b-c170-42bd-a7fb-b560be6650b3" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Deleted pod: open-rl-api-server-645ff9777d-zg9cl", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server-645ff9777d.18de9a78cd513e92", - "namespace": "openrl-system", - "resourceVersion": "1792039545387567001", - "uid": "cf4f8590-df28-4b16-81fc-f969ac56d830" - }, - "reason": "SuccessfulDelete", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546048591003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Successfully assigned openrl-system/open-rl-api-server-6d86b498f5-65swc to gke-open-rl-dra-default-pool-c19e0101-2bk0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a78f8298472", - "namespace": "openrl-system", - "resourceVersion": "1792039546102351012", - "uid": "dc59eb2c-80a6-4e27-84cb-c9e1ec4c59f0" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:49Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:49Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a79cecaeea8", - "namespace": "openrl-system", - "resourceVersion": "1792039549712335019", - "uid": "f3058d2c-fa64-40cd-b384-d338c7eb0bb8" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-api-server:coldstart1-dca985a\" in 9.007s (9.007s including waiting). Image size: 195425445 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7be7af61a3", - "namespace": "openrl-system", - "resourceVersion": "1792039558721215017", - "uid": "509c4369-d7f8-40c5-9eec-bcfd89092537" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7beba9be67", - "namespace": "openrl-system", - "resourceVersion": "1792039558783199013", - "uid": "923a6cf0-e083-4317-93f1-51e15f299e60" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:58Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{api-server}", - "kind": "Pod", - "name": "open-rl-api-server-6d86b498f5-65swc", - "namespace": "openrl-system", - "resourceVersion": "1792039546075535003", - "uid": "6e1edc03-0aef-4a5b-8ed9-a1839f82057b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:58Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:58Z", - "name": "open-rl-api-server-6d86b498f5-65swc.18de9a7bf0a5e517", - "namespace": "openrl-system", - "resourceVersion": "1792039558862303009", - "uid": "6926ae37-f0bc-46ee-b1b8-e597aaba7e60" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-default-pool-c19e0101-2bk0", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-default-pool-c19e0101-2bk0" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:46Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "open-rl-api-server-6d86b498f5", - "namespace": "openrl-system", - "resourceVersion": "1792039545940671018", - "uid": "638d78f3-dfa9-48e1-9a44-2e27cab2c0ad" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:46Z", - "message": "Created pod: open-rl-api-server-6d86b498f5-65swc", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:46Z", - "name": "open-rl-api-server-6d86b498f5.18de9a78f5fd4167", - "namespace": "openrl-system", - "resourceVersion": "1792039546068975009", - "uid": "badb8f80-576c-4eca-a4b5-e3e7a18dae6c" - }, - "reason": "SuccessfulCreate", - "reportingComponent": "replicaset-controller", - "reportingInstance": "", - "source": { - "component": "replicaset-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545232655018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled down replica set open-rl-api-server-645ff9777d from 1 to 0", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78c9668cf7", - "namespace": "openrl-system", - "resourceVersion": "1792039545327119005", - "uid": "7b1433df-3524-478b-aba2-8979950e245e" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:45:45Z", - "involvedObject": { - "apiVersion": "apps/v1", - "kind": "Deployment", - "name": "open-rl-api-server", - "namespace": "openrl-system", - "resourceVersion": "1792039545464319018", - "uid": "f05066e1-6b69-4415-a030-2f23afdeb965" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:45:45Z", - "message": "Scaled up replica set open-rl-api-server-6d86b498f5 from 0 to 1", - "metadata": { - "creationTimestamp": "2026-10-15T04:45:45Z", - "name": "open-rl-api-server.18de9a78ef8b9e1b", - "namespace": "openrl-system", - "resourceVersion": "1792039545956079020", - "uid": "627e6c0a-54b9-4b19-ac3a-9fdb2bf39a59" - }, - "reason": "ScalingReplicaSet", - "reportingComponent": "deployment-controller", - "reportingInstance": "", - "source": { - "component": "deployment-controller" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:19:18Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-google-gemma-4-e2b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041557994719022", - "uid": "36500320-9722-4016-9c5f-922d8e1491bb" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:19:18Z", - "message": "Successfully assigned openrl-system/orw-lora-google-gemma-4-e2b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:19:18Z", - "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4d6d1f6dbc", - "namespace": "openrl-system", - "resourceVersion": "1792041558108079006", - "uid": "553af22f-384f-4e70-98ea-6ea30397ee54" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:19:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-google-gemma-4-e2b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041558091711022", - "uid": "36500320-9722-4016-9c5f-922d8e1491bb" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:19:21Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:19:21Z", - "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4e5228875d", - "namespace": "openrl-system", - "resourceVersion": "1792041561952783019", - "uid": "9305ffbb-2e08-43ce-95fe-8365fe6ffc17" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:19:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-google-gemma-4-e2b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041558091711022", - "uid": "36500320-9722-4016-9c5f-922d8e1491bb" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:19:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:19:21Z", - "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4e52be3a84", - "namespace": "openrl-system", - "resourceVersion": "1792041561974879003", - "uid": "9ff5c7e2-efe0-49ab-8b33-ac55b4ed4f9b" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:19:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-google-gemma-4-e2b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041558091711022", - "uid": "36500320-9722-4016-9c5f-922d8e1491bb" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:19:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:19:22Z", - "name": "orw-lora-google-gemma-4-e2b-0-trainer.18de9c4e576b76a1", - "namespace": "openrl-system", - "resourceVersion": "1792041562044143010", - "uid": "1a78c2b9-2440-496b-b893-a5460a0b0cf9" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:15Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875490623001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:15Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:15Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac5b04afa67", - "namespace": "openrl-system", - "resourceVersion": "1792039875609279013", - "uid": "b012253c-b415-4a2d-94ef-c65232129e5c" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:51:19Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:51:19Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:51:19Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9ac681a006c9", - "namespace": "openrl-system", - "resourceVersion": "1792039879126319008", - "uid": "66ffdb37-2c85-49cd-b216-e45f392e57db" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 3m2.5s (3m2.5s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af0ff852ba6", - "namespace": "openrl-system", - "resourceVersion": "1792040061645247020", - "uid": "f5922761-c926-4e47-b8bd-c11949e587f4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:21Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:21Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:21Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af10d0803da", - "namespace": "openrl-system", - "resourceVersion": "1792040061849391002", - "uid": "cf5a12b5-3212-4bf2-aafa-161476e4d384" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:54:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:54:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:54:22Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9af124dfcd3f", - "namespace": "openrl-system", - "resourceVersion": "1792040062252079023", - "uid": "c1c4e655-706f-4620-a4ed-a094e4626eb0" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792039875591167001", - "uid": "cd8c3664-4bb0-4771-82cd-9a98ecb7515b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b0f88340550", - "namespace": "openrl-system", - "resourceVersion": "1792040192780783021", - "uid": "f724e2b7-bf58-4677-96ed-077a305f31ef" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264403887001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "0/11 nodes are available: 1 node(s) had untolerated taint(s), 10 node(s) didn't match Pod's node affinity/selector. still not schedulable, preemption: 0/11 nodes are available: 11 Preemption is not helpful for scheduling.", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2037f1e4ca", - "namespace": "openrl-system", - "resourceVersion": "1792040264444463021", - "uid": "a85006a2-68a6-4919-b148-08d58b546d07" - }, - "reason": "FailedScheduling", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Warning" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:44Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264434463001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:44Z", - "message": "Pod didn't trigger scale-up: 1 node(s) had untolerated taint(s)", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b2052291b90", - "namespace": "openrl-system", - "resourceVersion": "1792040264884223002", - "uid": "b1e9874c-4be8-4fc8-a4de-da91887af6a0" - }, - "reason": "NotTriggerScaleUp", - "reportingComponent": "cluster-autoscaler", - "reportingInstance": "", - "source": { - "component": "cluster-autoscaler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:13Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040264893039001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:13Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:13Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b26e90d9a95", - "namespace": "openrl-system", - "resourceVersion": "1792040293171567001", - "uid": "20b80848-a207-40e5-91b8-7d1cf5e515f8" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27b9848dbb", - "namespace": "openrl-system", - "resourceVersion": "1792040296672127014", - "uid": "49cee4e9-8387-4218-9033-ca33d35d7df9" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27bb7efef3", - "namespace": "openrl-system", - "resourceVersion": "1792040296703007017", - "uid": "66759db1-ec9d-4460-9910-cd26d8872e6a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:58:16Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:58:16Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:58:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b27c6957333", - "namespace": "openrl-system", - "resourceVersion": "1792040296889487014", - "uid": "85185869-c144-4d8b-bb78-50c6e3aa3d36" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040293156015001", - "uid": "570d2f8f-3f53-4819-bae7-d9abd474d51d" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b4619617ed0", - "namespace": "openrl-system", - "resourceVersion": "1792040427147711019", - "uid": "cc4da270-def2-456f-9374-372be8c717e4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:38Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498635999001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:38Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-sampler to gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:38Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b56c67c4be2", - "namespace": "openrl-system", - "resourceVersion": "1792040498767471000", - "uid": "234770ce-30d6-41da-bd67-f1d2dc1eb0ef" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b579823a5fc", - "namespace": "openrl-system", - "resourceVersion": "1792040502270255017", - "uid": "d015872c-dadd-4c0d-92a7-c58e778fb81c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b5799d1d2da", - "namespace": "openrl-system", - "resourceVersion": "1792040502296943006", - "uid": "68c5583e-0247-4418-a651-b4c1ec8f9623" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:42Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:42Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:42Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b57a48a02dd", - "namespace": "openrl-system", - "resourceVersion": "1792040502475407023", - "uid": "4fa3a7ac-047e-44ca-90ad-644364d8b890" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:09Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040498732799001", - "uid": "86ef51d2-867a-4a99-a8c4-fa843e3fd0ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:09Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:09Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-sampler.18de9b79f90c3c7c", - "namespace": "openrl-system", - "resourceVersion": "1792040649924575012", - "uid": "0f8bcc77-262c-4f25-8622-0875b9d390c2" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:52Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672220015012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:52Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:52Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a965bd0f797", - "namespace": "openrl-system", - "resourceVersion": "1792039672350591019", - "uid": "6fbb74be-86fe-4772-9530-ed9e714ea785" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:47:56Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:47:56Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T04:47:56Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9a9740750c3c", - "namespace": "openrl-system", - "resourceVersion": "1792039676166239015", - "uid": "c0108880-60ff-4b73-867a-1a229bae2d21" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 2m48.271s (2m48.271s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe6e35750f", - "namespace": "openrl-system", - "resourceVersion": "1792039844457167000", - "uid": "a050dc54-af84-4649-b74a-3f40e9bcb8c6" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:44Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:44Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:44Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe79a71089", - "namespace": "openrl-system", - "resourceVersion": "1792039844632591021", - "uid": "b4e1270d-6bd8-457d-9e80-7f3956beefe9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:50:45Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:50:45Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:50:45Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9abe9a14cead", - "namespace": "openrl-system", - "resourceVersion": "1792039845170959016", - "uid": "96f42f08-a549-49e5-82a7-b93812a007c1" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:56:32Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792039672311359012", - "uid": "1e83cdd2-a05f-46f8-886d-76e73c7c4651" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:56:32Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:56:32Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b0f95051228", - "namespace": "openrl-system", - "resourceVersion": "1792040192981183013", - "uid": "e1a61023-6c43-4653-b7e5-e5d827ae26d3" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:23Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040242926223012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:23Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:23Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1b3cdfef86", - "namespace": "openrl-system", - "resourceVersion": "1792040243043743024", - "uid": "c5640189-d94a-427a-9801-1c31b91f8c7f" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0dec271b", - "namespace": "openrl-system", - "resourceVersion": "1792040246545487023", - "uid": "a4c4d96d-d344-41b2-a4d1-b6517c70f8d7" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c0fbc3a46", - "namespace": "openrl-system", - "resourceVersion": "1792040246576895006", - "uid": "f1c9a5ae-1541-435b-a3ea-40a0b4fad1b6" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:57:26Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:57:26Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:57:26Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b1c198fbeb0", - "namespace": "openrl-system", - "resourceVersion": "1792040246742815002", - "uid": "e3c191ab-8027-4ac0-bd64-a6a183462a78" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:00:27Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040243022511012", - "uid": "b4b9c50b-214c-4078-8ab9-8b8479a48a71" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:00:27Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:00:27Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b4622b19144", - "namespace": "openrl-system", - "resourceVersion": "1792040427285631023", - "uid": "f846e92c-c091-4b33-8f8d-184c509fc379" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:16Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476805407012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:16Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen2-5-0-5b-0-trainer to gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:16Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b51b2221f21", - "namespace": "openrl-system", - "resourceVersion": "1792040476942623014", - "uid": "d1e74df5-882f-4a98-9411-b9cfcc62aaea" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5294e764a1", - "namespace": "openrl-system", - "resourceVersion": "1792040480747055005", - "uid": "44dddc5e-ab5e-4af2-9b5f-647a90de018f" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b5297c2c92a", - "namespace": "openrl-system", - "resourceVersion": "1792040480787967019", - "uid": "8f7934b7-adde-4067-b4ba-4a997a7588cb" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:01:20Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:01:20Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:01:20Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b52a24a154b", - "namespace": "openrl-system", - "resourceVersion": "1792040480963103007", - "uid": "d3158913-c79e-43d4-b468-d748641d49fe" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:10Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040476916127012", - "uid": "511519fc-c49c-4bb7-9aec-6d79e58c6844" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:10Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:10Z", - "name": "orw-lora-qwen-qwen2-5-0-5b-0-trainer.18de9b7a08fbdf5f", - "namespace": "openrl-system", - "resourceVersion": "1792040650191935010", - "uid": "4e13aefe-c0ba-4c9d-b65f-61d9e60845e5" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588098239023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9999f1f5d1f3", - "namespace": "openrl-system", - "resourceVersion": "1792038588224095005", - "uid": "0b8a92c0-6ac3-4582-ab5f-844d4ffa7b31" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad07e7f58", - "namespace": "openrl-system", - "resourceVersion": "1792038591977631020", - "uid": "9bb08ab0-c215-452c-ac24-0b011c104083" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:51Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:51Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:51Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad104508d", - "namespace": "openrl-system", - "resourceVersion": "1792038592011567002", - "uid": "cebf5c03-9d69-4cae-95ec-b96ae6cc20e9" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:29:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:29:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:29:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de999ad5fe5f28", - "namespace": "openrl-system", - "resourceVersion": "1792038592051215017", - "uid": "d431fad3-bafe-4002-8042-480b29aa499b" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792038588203679023", - "uid": "8890e7fc-b67b-4b1b-b3a2-94806b6a0581" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de99d392107192", - "namespace": "openrl-system", - "resourceVersion": "1792038835732623016", - "uid": "0176ffb8-1f99-4aa3-94ab-51b845b7dbee" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:58Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818213471023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:58Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:58Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba133bf0551", - "namespace": "openrl-system", - "resourceVersion": "1792040818411407005", - "uid": "d40f130e-4611-48a7-a8cf-475ea874742b" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:07:01Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:07:01Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:07:01Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9ba20157b461", - "namespace": "openrl-system", - "resourceVersion": "1792040821861695012", - "uid": "6c78cc31-3a5b-4a65-921e-6d2413d11847" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.459s (1m26.459s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb622c15bdb", - "namespace": "openrl-system", - "resourceVersion": "1792040908347887024", - "uid": "60bbe5b5-d0aa-4c23-a86c-6f68d159450c" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb6236dbe8e", - "namespace": "openrl-system", - "resourceVersion": "1792040908367439012", - "uid": "1efb0578-38c6-452f-bb34-e21ce62dd0ed" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:08:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:08:28Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:08:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bb62857081d", - "namespace": "openrl-system", - "resourceVersion": "1792040908418671004", - "uid": "d44c3b3f-3e97-42f9-9a2c-c5eb1f6cd987" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:10:57Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792040818390799023", - "uid": "6eacfcdb-f671-4b04-8807-45530a74969b" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:10:57Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:10:57Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bd8cbffc810", - "namespace": "openrl-system", - "resourceVersion": "1792041057212047016", - "uid": "4a67fe5e-01b5-4b84-9453-b20887271ae4" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:18Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138729647023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:18Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:18Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bebcffc9f4c", - "namespace": "openrl-system", - "resourceVersion": "1792041138871999014", - "uid": "3e19d3e2-4122-4f9d-8324-8f74961c36bc" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9c8a8459", - "namespace": "openrl-system", - "resourceVersion": "1792041142294159019", - "uid": "aaaf55ef-ec50-4fa5-b6a7-28a27d836f8e" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9bec9d08793e", - "namespace": "openrl-system", - "resourceVersion": "1792041142311631019", - "uid": "5ab187bd-863a-4013-9e2d-185b71c68ead" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:12:22Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:12:22Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:12:22Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9beca1f20c38", - "namespace": "openrl-system", - "resourceVersion": "1792041142412303013", - "uid": "193a6a12-3e5a-44e3-8d69-ea9d46ee4707" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:14:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041138843567023", - "uid": "0feae45d-703a-45d4-a16d-e3c84947606f" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:14:49Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:14:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c0ec69a2183", - "namespace": "openrl-system", - "resourceVersion": "1792041289046591010", - "uid": "7161d68e-6e82-48a1-894f-7f149cadb6db" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:09Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369352463023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:09Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-sampler to gke-open-rl-dra-h100-sampler-a721f127-0853", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:09Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2181d762b3", - "namespace": "openrl-system", - "resourceVersion": "1792041369496943024", - "uid": "6a40425d-424d-438f-b398-45410b5817eb" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:13Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369459823023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:13Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:13Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c22606a262a", - "namespace": "openrl-system", - "resourceVersion": "1792041373214223010", - "uid": "c67940e2-268e-4a9a-94d4-05be2058728b" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:13Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369459823023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:13Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:13Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2260e74a63", - "namespace": "openrl-system", - "resourceVersion": "1792041373233151005", - "uid": "ecec41f0-1307-4892-ad43-57226155574a" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:16:13Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369459823023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:16:13Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:16:13Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c2266080366", - "namespace": "openrl-system", - "resourceVersion": "1792041373308543020", - "uid": "6dc932f6-ae1f-48f6-b85d-e087521c8e20" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:18:28Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-sampler", - "namespace": "openrl-system", - "resourceVersion": "1792041369459823023", - "uid": "52b0bb8c-7247-429f-99f3-c08a2a76d251" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:18:28Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:18:28Z", - "name": "orw-lora-qwen-qwen3-8b-0-sampler.18de9c41f90718e5", - "namespace": "openrl-system", - "resourceVersion": "1792041508933663010", - "uid": "5c52845b-38a0-49b7-a3f5-f00f82aa7589" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-sampler-a721f127-0853", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-sampler-a721f127-0853" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:48Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528865007006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:48Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:48Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998c27205a21", - "namespace": "openrl-system", - "resourceVersion": "1792038529003663020", - "uid": "52a2d90f-b6da-4570-8e28-b0b726d202e4" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:agent3-9f8dc13\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0660e64d", - "namespace": "openrl-system", - "resourceVersion": "1792038532754447004", - "uid": "8b75e1eb-439d-487d-8398-87c7b9701816" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d06e5b28d", - "namespace": "openrl-system", - "resourceVersion": "1792038532776847006", - "uid": "d7fdaab9-c6a8-4e67-882c-cdd1eff1c685" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:28:52Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:28:52Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T04:28:52Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de998d0bc39234", - "namespace": "openrl-system", - "resourceVersion": "1792038532827151022", - "uid": "8d85e4c6-7bfd-4acb-9966-e263d5fdd567" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T04:33:55Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792038528965487006", - "uid": "e19ca60b-e37f-4763-a909-97fb3c20e739" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T04:33:55Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T04:33:55Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de99d37e940b3e", - "namespace": "openrl-system", - "resourceVersion": "1792038835415695012", - "uid": "918abc9a-1f24-4767-8928-b202482f11ed" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:04:59Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699253647006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:04:59Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:04:59Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b857d3e96e6", - "namespace": "openrl-system", - "resourceVersion": "1792040699411615020", - "uid": "d8e12f66-cfb5-44e7-806c-62f7dfdf53be" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:05:02Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:05:02Z", - "message": "Pulling image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\"", - "metadata": { - "creationTimestamp": "2026-10-15T05:05:02Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b864b7869c2", - "namespace": "openrl-system", - "resourceVersion": "1792040702849391020", - "uid": "192a187a-51a0-4020-aa5a-1d23ca35e747" - }, - "reason": "Pulling", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Successfully pulled image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" in 1m26.476s (1m26.476s including waiting). Image size: 7141761889 bytes.", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6de6ca39", - "namespace": "openrl-system", - "resourceVersion": "1792040789347727015", - "uid": "7aa94cd4-02c5-494d-92df-ff88f0fb46a4" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a6e7f2537", - "namespace": "openrl-system", - "resourceVersion": "1792040789380863020", - "uid": "5c982773-7a8b-4081-8db2-4eef4939d5da" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:06:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:06:29Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:06:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9b9a735c340c", - "namespace": "openrl-system", - "resourceVersion": "1792040789417615006", - "uid": "0973fe5b-e1d0-4cff-9e2f-fe6233ca6d77" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:10:57Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792040699366367006", - "uid": "3e8a323d-44af-454a-b8e2-93793af914d6" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:10:57Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:10:57Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9bd8d38751ab", - "namespace": "openrl-system", - "resourceVersion": "1792041057319999002", - "uid": "0af06e74-e0c1-44de-a612-6fba47fb2b5c" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:46Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106258495006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:46Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:46Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be43ff3ded1", - "namespace": "openrl-system", - "resourceVersion": "1792041106381007003", - "uid": "4842f6c0-d606-444b-a970-6e9484189707" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50d27add6", - "namespace": "openrl-system", - "resourceVersion": "1792041109821407010", - "uid": "4e9749a9-e70a-453f-b51f-4d5ea7a60e86" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be50daf2189", - "namespace": "openrl-system", - "resourceVersion": "1792041109840015005", - "uid": "2950a976-a558-471f-94fb-1ac680004e4c" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:11:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:11:49Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:11:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9be5134afada", - "namespace": "openrl-system", - "resourceVersion": "1792041109931471020", - "uid": "d1c0b2ae-50a0-41e8-8333-441dd2250b23" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:14:49Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041106361663006", - "uid": "f3fb2bbe-27c2-4585-912e-05837e6775ff" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:14:49Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:14:49Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c0ece29f9b7", - "namespace": "openrl-system", - "resourceVersion": "1792041289153695008", - "uid": "9db24059-5412-4998-8f1e-ff4723a02422" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:37Z", - "involvedObject": { - "apiVersion": "v1", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337881599006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:37Z", - "message": "Successfully assigned openrl-system/orw-lora-qwen-qwen3-8b-0-trainer to gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:37Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1a2ddca97f", - "namespace": "openrl-system", - "resourceVersion": "1792041338007727010", - "uid": "622520b7-a0c6-4899-9d8e-8b7e289a5c19" - }, - "reason": "Scheduled", - "reportingComponent": "default-scheduler", - "reportingInstance": "", - "source": { - "component": "default-scheduler" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337986559006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:41Z", - "message": "Container image \"us-central1-docker.pkg.dev/cdrollouts-sunilarora/open-rl/open-rl-server:coldstart1-dca985a\" already present on machine and can be accessed by the pod", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:41Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afae88647", - "namespace": "openrl-system", - "resourceVersion": "1792041341445023014", - "uid": "174539ca-68ae-411a-9602-b94910a4f8a1" - }, - "reason": "Pulled", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337986559006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:41Z", - "message": "Container created", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:41Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1afb72b4d7", - "namespace": "openrl-system", - "resourceVersion": "1792041341467327017", - "uid": "abc887ed-fbc2-4086-9892-21b9dbd9ef25" - }, - "reason": "Created", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:15:41Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337986559006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:15:41Z", - "message": "Container started", - "metadata": { - "creationTimestamp": "2026-10-15T05:15:41Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c1b0091bf79", - "namespace": "openrl-system", - "resourceVersion": "1792041341538399007", - "uid": "ce445484-5b88-4ca9-a93d-806ea29b103f" - }, - "reason": "Started", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - }, - { - "apiVersion": "v1", - "count": 1, - "eventTime": null, - "firstTimestamp": "2026-10-15T05:18:29Z", - "involvedObject": { - "apiVersion": "v1", - "fieldPath": "spec.containers{worker}", - "kind": "Pod", - "name": "orw-lora-qwen-qwen3-8b-0-trainer", - "namespace": "openrl-system", - "resourceVersion": "1792041337986559006", - "uid": "0ee6fe0b-938b-4a40-b3fd-369206fef170" - }, - "kind": "Event", - "lastTimestamp": "2026-10-15T05:18:29Z", - "message": "Stopping container worker", - "metadata": { - "creationTimestamp": "2026-10-15T05:18:29Z", - "name": "orw-lora-qwen-qwen3-8b-0-trainer.18de9c41ffc5e74a", - "namespace": "openrl-system", - "resourceVersion": "1792041509032175007", - "uid": "aeee14b0-c22c-4f6d-b6c9-5f5dffbcc725" - }, - "reason": "Killing", - "reportingComponent": "kubelet", - "reportingInstance": "gke-open-rl-dra-h100-trainer-702095b4-1g3a", - "source": { - "component": "kubelet", - "host": "gke-open-rl-dra-h100-trainer-702095b4-1g3a" - }, - "type": "Normal" - } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" - } -} diff --git a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt b/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt deleted file mode 100644 index 167c1377..00000000 --- a/runs/2026-10-15_cold_start/after-E1-gemma-e2b-h100-trainer-only/log-orw-lora-google-gemma-4-e2b-0-trainer.txt +++ /dev/null @@ -1,32 +0,0 @@ -2026-10-15T05:19:26.404504520Z -2026-10-15T05:19:26.404565293Z ================================================== -2026-10-15T05:19:26.404587125Z Open-RL PyTorch Training Worker -2026-10-15T05:19:26.404588917Z ================================================== -2026-10-15T05:19:26.404590495Z -> Hardware : CUDA_VISIBLE_DEVICES=ALL -2026-10-15T05:19:26.421803817Z -> Fine-Tuning Type: lora (Is LoRA: True) -2026-10-15T05:19:26.421826430Z -2026-10-15T05:19:26.629647334Z Loading base model google/gemma-4-e2b to cuda:0... -2026-10-15T05:19:27.261237071Z Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. -2026-10-15T05:19:31.643943906Z [prefetch] reading 1 file(s), 9.54 GiB, from nfs into the page cache with 8 threads -2026-10-15T05:20:56.741663012Z [prefetch] done: 9.54 GiB in 85.1 s (115 MiB/s) -2026-10-15T05:20:58.914432483Z -Loading weights: 0%| | 0/1951 [00:00 "$R/$1.run.log" 2>&1; echo "$1 DONE=$? $(date -u +%T)" >> "$R/after-progress.txt"; } -run after-A1-0.5b-l4-cold Qwen/Qwen2.5-0.5B lvw8 2208 -run after-B1-0.5b-l4-cached Qwen/Qwen2.5-0.5B lvw8 2208 -run after-B2-0.5b-l4-cached Qwen/Qwen2.5-0.5B lvw8 2208 -run after-C1-8b-h100-cold Qwen/Qwen3-8B 1g3a 0853 -run after-D1-8b-h100-cached Qwen/Qwen3-8B 1g3a 0853 -run after-D2-8b-h100-cached Qwen/Qwen3-8B 1g3a 0853 -echo "ALL DONE $(date -u +%T)" >> "$R/after-progress.txt" diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt deleted file mode 100644 index 4c02098a..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/file.txt +++ /dev/null @@ -1 +0,0 @@ -file: /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json deleted file mode 100644 index 10888371..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-randread-4k-direct.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039384, - "timestamp_ms" : 1792039384251, - "time" : "Thu Oct 15 04:43:04 2026", - "jobs" : [ - { - "jobname" : "randread", - "groupid" : 0, - "job_start" : 1792039354208, - "error" : 0, - "eta" : 0, - "elapsed" : 31, - "job options" : { - "name" : "randread", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", - "rw" : "randread", - "bs" : "4k", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "30", - "time_based" : "1" - }, - "read" : { - "io_bytes" : 4075520, - "io_kbytes" : 3980, - "bw_bytes" : 135714, - "bw" : 132, - "iops" : 33.133533, - "runtime" : 30030, - "total_ios" : 995, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 894793, - "max" : 576275926, - "mean" : 30175894.901508, - "stddev" : 43069178.925758, - "N" : 995, - "percentile" : { - "1.000000" : 913408, - "5.000000" : 962560, - "10.000000" : 995328, - "20.000000" : 1036288, - "30.000000" : 9764864, - "40.000000" : 20578304, - "50.000000" : 24772608, - "60.000000" : 29491200, - "70.000000" : 35389440, - "80.000000" : 43253760, - "90.000000" : 61603840, - "95.000000" : 77070336, - "99.000000" : 152043520, - "99.500000" : 240123904, - "99.900000" : 574619648, - "99.950000" : 574619648, - "99.990000" : 574619648 - } - }, - "lat_ns" : { - "min" : 895008, - "max" : 576276158, - "mean" : 30176112.192965, - "stddev" : 43069176.870365, - "N" : 995 - }, - "bw_min" : 16, - "bw_max" : 232, - "bw_agg" : 99.597683, - "bw_mean" : 132.533333, - "bw_dev" : 46.352908, - "bw_samples" : 60, - "iops_min" : 4, - "iops_max" : 58, - "iops_mean" : 33.133333, - "iops_stddev" : 11.588227, - "iops_samples" : 60 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 30029, - "usr_cpu" : 0.013320, - "sys_cpu" : 0.073263, - "ctx" : 996, - "majf" : 0, - "minf" : 9, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 11.356784 - }, - "latency_ms" : { - "2" : 17.185930, - "4" : 1.206030, - "10" : 0.301508, - "20" : 8.542714, - "50" : 45.427136, - "100" : 13.065327, - "250" : 2.512563, - "500" : 0.000000, - "750" : 0.402010, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json deleted file mode 100644 index a7ac7b43..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-4jobs-direct.json +++ /dev/null @@ -1,261 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039447, - "timestamp_ms" : 1792039447569, - "time" : "Thu Oct 15 04:44:07 2026", - "jobs" : [ - { - "jobname" : "seqread-1M-4jobs", - "groupid" : 0, - "job_start" : 1792039417469, - "error" : 0, - "eta" : 0, - "elapsed" : 31, - "job options" : { - "name" : "seqread-1M-4jobs", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", - "rw" : "read", - "bs" : "1M", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "4", - "offset_increment" : "20%", - "runtime" : "30", - "time_based" : "1" - }, - "read" : { - "io_bytes" : 4250927104, - "io_kbytes" : 4151296, - "bw_bytes" : 141292531, - "bw" : 137980, - "iops" : 134.747058, - "runtime" : 30086, - "total_ios" : 4054, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 1172703, - "max" : 943811245, - "mean" : 29659560.486186, - "stddev" : 53937635.074449, - "N" : 4054, - "percentile" : { - "1.000000" : 1220608, - "5.000000" : 1269760, - "10.000000" : 1318912, - "20.000000" : 1400832, - "30.000000" : 1548288, - "40.000000" : 1957888, - "50.000000" : 2637824, - "60.000000" : 7438336, - "70.000000" : 27656192, - "80.000000" : 56360960, - "90.000000" : 96993280, - "95.000000" : 135266304, - "99.000000" : 217055232, - "99.500000" : 263192576, - "99.900000" : 541065216, - "99.950000" : 574619648, - "99.990000" : 943718400 - } - }, - "lat_ns" : { - "min" : 1172739, - "max" : 943811484, - "mean" : 29659692.227430, - "stddev" : 53937691.104136, - "N" : 4054 - }, - "bw_min" : 10240, - "bw_max" : 1286144, - "bw_agg" : 100.000000, - "bw_mean" : 138337.771751, - "bw_dev" : 90992.521392, - "bw_samples" : 238, - "iops_min" : 10, - "iops_max" : 1256, - "iops_mean" : 135.095480, - "iops_stddev" : 88.859884, - "iops_samples" : 238 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 120255, - "usr_cpu" : 0.007484, - "sys_cpu" : 0.212049, - "ctx" : 4064, - "majf" : 0, - "minf" : 1061, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_ms" : { - "2" : 40.083868, - "4" : 13.394179, - "10" : 11.198816, - "20" : 3.305377, - "50" : 10.039467, - "100" : 12.629502, - "250" : 8.658115, - "500" : 0.518007, - "750" : 0.148002, - "1000" : 0.024667, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json deleted file mode 100644 index 5f1e6a58..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-buffered.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039509, - "timestamp_ms" : 1792039509047, - "time" : "Thu Oct 15 04:45:09 2026", - "jobs" : [ - { - "jobname" : "seqread-buffered", - "groupid" : 0, - "job_start" : 1792039448961, - "error" : 0, - "eta" : 0, - "elapsed" : 61, - "job options" : { - "name" : "seqread-buffered", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", - "rw" : "read", - "bs" : "1M", - "direct" : "0", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "60", - "time_based" : "0" - }, - "read" : { - "io_bytes" : 3805282304, - "io_kbytes" : 3716096, - "bw_bytes" : 63342193, - "bw" : 61857, - "iops" : 60.407824, - "runtime" : 60075, - "total_ios" : 3629, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 100093, - "max" : 1184710687, - "mean" : 16551942.435382, - "stddev" : 46828063.988788, - "N" : 3629, - "percentile" : { - "1.000000" : 119296, - "5.000000" : 152576, - "10.000000" : 276480, - "20.000000" : 460800, - "30.000000" : 692224, - "40.000000" : 798720, - "50.000000" : 954368, - "60.000000" : 1089536, - "70.000000" : 1318912, - "80.000000" : 11337728, - "90.000000" : 58458112, - "95.000000" : 99090432, - "99.000000" : 179306496, - "99.500000" : 229638144, - "99.900000" : 591396864, - "99.950000" : 633339904, - "99.990000" : 1182793728 - } - }, - "lat_ns" : { - "min" : 100311, - "max" : 1184711128, - "mean" : 16552079.251309, - "stddev" : 46828089.869993, - "N" : 3629 - }, - "bw_min" : 2048, - "bw_max" : 1347584, - "bw_agg" : 100.000000, - "bw_mean" : 62438.184874, - "bw_dev" : 202926.862845, - "bw_samples" : 119, - "iops_min" : 2, - "iops_max" : 1316, - "iops_mean" : 60.974790, - "iops_stddev" : 198.170764, - "iops_samples" : 119 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 60074, - "usr_cpu" : 0.019975, - "sys_cpu" : 0.759064, - "ctx" : 3357, - "majf" : 0, - "minf" : 268, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 8.845412, - "500" : 13.585010, - "750" : 11.380546, - "1000" : 20.088179 - }, - "latency_ms" : { - "2" : 19.977955, - "4" : 1.598236, - "10" : 4.298705, - "20" : 2.011573, - "50" : 6.200055, - "100" : 7.109397, - "250" : 4.519151, - "500" : 0.275558, - "750" : 0.082667, - "1000" : 0.000000, - "2000" : 0.027556, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json deleted file mode 100644 index 943ffb14..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-1M-direct.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039352, - "timestamp_ms" : 1792039352765, - "time" : "Thu Oct 15 04:42:32 2026", - "jobs" : [ - { - "jobname" : "seqread", - "groupid" : 0, - "job_start" : 1792039292506, - "error" : 0, - "eta" : 0, - "elapsed" : 61, - "job options" : { - "name" : "seqread", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", - "rw" : "read", - "bs" : "1M", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "60", - "time_based" : "0" - }, - "read" : { - "io_bytes" : 1023410176, - "io_kbytes" : 999424, - "bw_bytes" : 16986342, - "bw" : 16588, - "iops" : 16.199439, - "runtime" : 60249, - "total_ios" : 976, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 5615855, - "max" : 738292284, - "mean" : 61723383.765369, - "stddev" : 60353873.274478, - "N" : 976, - "percentile" : { - "1.000000" : 6324224, - "5.000000" : 6914048, - "10.000000" : 7569408, - "20.000000" : 10027008, - "30.000000" : 26345472, - "40.000000" : 35913728, - "50.000000" : 47972352, - "60.000000" : 61603840, - "70.000000" : 76021760, - "80.000000" : 100139008, - "90.000000" : 128450560, - "95.000000" : 162529280, - "99.000000" : 238026752, - "99.500000" : 371195904, - "99.900000" : 742391808, - "99.950000" : 742391808, - "99.990000" : 742391808 - } - }, - "lat_ns" : { - "min" : 5616100, - "max" : 738292769, - "mean" : 61723666.015369, - "stddev" : 60353875.885652, - "N" : 976 - }, - "bw_min" : 4096, - "bw_max" : 36864, - "bw_agg" : 100.000000, - "bw_mean" : 16640.000000, - "bw_dev" : 6235.464846, - "bw_samples" : 120, - "iops_min" : 4, - "iops_max" : 36, - "iops_mean" : 16.250000, - "iops_stddev" : 6.089321, - "iops_samples" : 120 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 60248, - "usr_cpu" : 0.006639, - "sys_cpu" : 0.129465, - "ctx" : 978, - "majf" : 0, - "minf" : 269, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_ms" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 19.774590, - "20" : 6.762295, - "50" : 25.102459, - "100" : 28.073770, - "250" : 19.467213, - "500" : 0.614754, - "750" : 0.204918, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json deleted file mode 100644 index 430aa12a..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/fio-seqread-256k-qd1-direct.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039416, - "timestamp_ms" : 1792039416020, - "time" : "Thu Oct 15 04:43:36 2026", - "jobs" : [ - { - "jobname" : "seqread-256k-qd1", - "groupid" : 0, - "job_start" : 1792039385969, - "error" : 0, - "eta" : 0, - "elapsed" : 31, - "job options" : { - "name" : "seqread-256k-qd1", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors", - "rw" : "read", - "bs" : "256k", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "30", - "time_based" : "1" - }, - "read" : { - "io_bytes" : 1409810432, - "io_kbytes" : 1376768, - "bw_bytes" : 46945171, - "bw" : 45844, - "iops" : 179.081616, - "runtime" : 30031, - "total_ios" : 5378, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 985502, - "max" : 381577086, - "mean" : 5583040.162886, - "stddev" : 15453629.349114, - "N" : 5378, - "percentile" : { - "1.000000" : 1011712, - "5.000000" : 1044480, - "10.000000" : 1073152, - "20.000000" : 1105920, - "30.000000" : 1138688, - "40.000000" : 1171456, - "50.000000" : 1236992, - "60.000000" : 1335296, - "70.000000" : 1679360, - "80.000000" : 2473984, - "90.000000" : 6979584, - "95.000000" : 34340864, - "99.000000" : 76021760, - "99.500000" : 92798976, - "99.900000" : 147849216, - "99.950000" : 158334976, - "99.990000" : 379584512 - } - }, - "lat_ns" : { - "min" : 985546, - "max" : 381577157, - "mean" : 5583109.203979, - "stddev" : 15453645.045346, - "N" : 5378 - }, - "bw_min" : 5632, - "bw_max" : 229888, - "bw_agg" : 100.000000, - "bw_mean" : 45883.733333, - "bw_dev" : 70722.254239, - "bw_samples" : 60, - "iops_min" : 22, - "iops_max" : 898, - "iops_mean" : 179.233333, - "iops_stddev" : 276.258806, - "iops_samples" : 60 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 30030, - "usr_cpu" : 0.043290, - "sys_cpu" : 0.326340, - "ctx" : 5379, - "majf" : 0, - "minf" : 74, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.185943 - }, - "latency_ms" : { - "2" : 71.309037, - "4" : 18.185199, - "10" : 0.409074, - "20" : 1.283005, - "50" : 5.931573, - "100" : 2.361473, - "250" : 0.316103, - "500" : 0.018594, - "750" : 0.000000, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt deleted file mode 100644 index 191b93ff..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-after.txt +++ /dev/null @@ -1,31 +0,0 @@ -device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 - opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none - age: 232 - caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 - sec: flavor=1,pseudoflavor=1 - events: 31 451 0 0 13 9 518 0 0 3630 0 0 4 0 11 0 0 11 0 0 0 0 0 0 0 0 0 - bytes: 3805282304 0 6688223232 0 10494554112 0 929280 0 - RPC iostats version: 1.1 p/v: 100003/3 (nfs) - xprt: tcp 961 1 2 0 1 15104 15104 0 30403 0 4 13 15292 - per-op statistics - NULL: 1 1 0 44 24 3 1 4 0 - GETATTR: 33 33 0 4720 3696 0 34 34 0 - SETATTR: 0 0 0 0 0 0 0 0 0 - LOOKUP: 17 17 0 3100 4012 0 16 17 0 - ACCESS: 9 9 0 1324 1080 0 9 9 0 - READLINK: 3 3 0 432 588 0 38 38 0 - READ: 15033 15033 0 2345148 10496478336 206 346790 347214 0 - WRITE: 0 0 0 0 0 0 0 0 0 - CREATE: 0 0 0 0 0 0 0 0 0 - MKDIR: 0 0 0 0 0 0 0 0 0 - SYMLINK: 0 0 0 0 0 0 0 0 0 - MKNOD: 0 0 0 0 0 0 0 0 0 - REMOVE: 0 0 0 0 0 0 0 0 0 - RMDIR: 0 0 0 0 0 0 0 0 0 - RENAME: 0 0 0 0 0 0 0 0 0 - LINK: 0 0 0 0 0 0 0 0 0 - READDIR: 0 0 0 0 0 0 0 0 0 - READDIRPLUS: 2 2 0 336 2764 0 2 2 0 - FSSTAT: 2 2 0 272 168 0 1 1 0 - FSINFO: 2 2 0 272 160 0 2 2 0 - PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt b/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt deleted file mode 100644 index 499e4613..00000000 --- a/runs/2026-10-15_cold_start/fio-h100-1g3a-coldfile/mountstats-before.txt +++ /dev/null @@ -1,32 +0,0 @@ -lrwxrwxrwx 1 root root 76 Sep 17 20:59 /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554/model-00001-of-00003.safetensors -> ../../blobs/75311d91bb08cf0b882913da464a1e722a31fb44db35208663487efb7a3d8ed6 -device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 - opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none - age: 13 - caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 - sec: flavor=1,pseudoflavor=1 - events: 1 72 0 0 2 9 89 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - bytes: 0 0 0 0 0 0 0 0 - RPC iostats version: 1.1 p/v: 100003/3 (nfs) - xprt: tcp 961 1 2 0 1 30 30 0 30 0 2 0 0 - per-op statistics - NULL: 1 1 0 44 24 3 1 4 0 - GETATTR: 3 3 0 416 336 0 3 3 0 - SETATTR: 0 0 0 0 0 0 0 0 0 - LOOKUP: 9 9 0 1596 2124 0 8 8 0 - ACCESS: 8 8 0 1176 960 0 8 9 0 - READLINK: 3 3 0 432 588 0 38 38 0 - READ: 0 0 0 0 0 0 0 0 0 - WRITE: 0 0 0 0 0 0 0 0 0 - CREATE: 0 0 0 0 0 0 0 0 0 - MKDIR: 0 0 0 0 0 0 0 0 0 - SYMLINK: 0 0 0 0 0 0 0 0 0 - MKNOD: 0 0 0 0 0 0 0 0 0 - REMOVE: 0 0 0 0 0 0 0 0 0 - RMDIR: 0 0 0 0 0 0 0 0 0 - RENAME: 0 0 0 0 0 0 0 0 0 - LINK: 0 0 0 0 0 0 0 0 0 - READDIR: 0 0 0 0 0 0 0 0 0 - READDIRPLUS: 2 2 0 336 2764 0 2 2 0 - FSSTAT: 0 0 0 0 0 0 0 0 0 - FSINFO: 2 2 0 272 160 0 2 2 0 - PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt b/runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt deleted file mode 100644 index 9687ad99..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/file.txt +++ /dev/null @@ -1 +0,0 @@ -file: /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json deleted file mode 100644 index e80f7dbb..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-randread-4k-direct.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039170, - "timestamp_ms" : 1792039170876, - "time" : "Thu Oct 15 04:39:30 2026", - "jobs" : [ - { - "jobname" : "randread", - "groupid" : 0, - "job_start" : 1792039140863, - "error" : 0, - "eta" : 0, - "elapsed" : 31, - "job options" : { - "name" : "randread", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", - "rw" : "randread", - "bs" : "4k", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "30", - "time_based" : "1" - }, - "read" : { - "io_bytes" : 131043328, - "io_kbytes" : 127972, - "bw_bytes" : 4367965, - "bw" : 4265, - "iops" : 1066.397787, - "runtime" : 30001, - "total_ios" : 31993, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 809541, - "max" : 4640366, - "mean" : 936167.207389, - "stddev" : 85513.990026, - "N" : 31993, - "percentile" : { - "1.000000" : 839680, - "5.000000" : 856064, - "10.000000" : 872448, - "20.000000" : 880640, - "30.000000" : 897024, - "40.000000" : 905216, - "50.000000" : 921600, - "60.000000" : 937984, - "70.000000" : 954368, - "80.000000" : 978944, - "90.000000" : 1011712, - "95.000000" : 1056768, - "99.000000" : 1155072, - "99.500000" : 1204224, - "99.900000" : 1646592, - "99.950000" : 2277376, - "99.990000" : 3719168 - } - }, - "lat_ns" : { - "min" : 809614, - "max" : 4640441, - "mean" : 936290.783265, - "stddev" : 85520.922977, - "N" : 31993 - }, - "bw_min" : 3896, - "bw_max" : 4408, - "bw_agg" : 99.986149, - "bw_mean" : 4265.898305, - "bw_dev" : 104.891026, - "bw_samples" : 59, - "iops_min" : 974, - "iops_max" : 1102, - "iops_mean" : 1066.474576, - "iops_stddev" : 26.222757, - "iops_samples" : 59 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 30000, - "usr_cpu" : 0.253333, - "sys_cpu" : 1.473333, - "ctx" : 31994, - "majf" : 0, - "minf" : 11, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 86.890882 - }, - "latency_ms" : { - "2" : 13.049730, - "4" : 0.050011, - "10" : 0.010000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json deleted file mode 100644 index 9dfff85b..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-4jobs-direct.json +++ /dev/null @@ -1,261 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039233, - "timestamp_ms" : 1792039233725, - "time" : "Thu Oct 15 04:40:33 2026", - "jobs" : [ - { - "jobname" : "seqread-1M-4jobs", - "groupid" : 0, - "job_start" : 1792039203706, - "error" : 0, - "eta" : 0, - "elapsed" : 31, - "job options" : { - "name" : "seqread-1M-4jobs", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", - "rw" : "read", - "bs" : "1M", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "4", - "offset_increment" : "20%", - "runtime" : "30", - "time_based" : "1" - }, - "read" : { - "io_bytes" : 55853449216, - "io_kbytes" : 54544384, - "bw_bytes" : 1861595480, - "bw" : 1817964, - "iops" : 1775.355798, - "runtime" : 30003, - "total_ios" : 53266, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 1526779, - "max" : 21354553, - "mean" : 2249205.334566, - "stddev" : 1263897.589092, - "N" : 53266, - "percentile" : { - "1.000000" : 1859584, - "5.000000" : 2007040, - "10.000000" : 2039808, - "20.000000" : 2072576, - "30.000000" : 2088960, - "40.000000" : 2113536, - "50.000000" : 2113536, - "60.000000" : 2146304, - "70.000000" : 2146304, - "80.000000" : 2179072, - "90.000000" : 2211840, - "95.000000" : 2244608, - "99.000000" : 5210112, - "99.500000" : 15138816, - "99.900000" : 19005440, - "99.950000" : 19791872, - "99.990000" : 20316160 - } - }, - "lat_ns" : { - "min" : 1526935, - "max" : 21354747, - "mean" : 2249392.502103, - "stddev" : 1263898.426861, - "N" : 53266 - }, - "bw_min" : 1267712, - "bw_max" : 1939456, - "bw_agg" : 100.000000, - "bw_mean" : 1819144.677966, - "bw_dev" : 38366.646123, - "bw_samples" : 236, - "iops_min" : 1238, - "iops_max" : 1894, - "iops_mean" : 1776.508475, - "iops_stddev" : 37.467428, - "iops_samples" : 236 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 120004, - "usr_cpu" : 0.233326, - "sys_cpu" : 3.800707, - "ctx" : 53319, - "majf" : 0, - "minf" : 1077, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_ms" : { - "2" : 4.126460, - "4" : 94.531221, - "10" : 0.563211, - "20" : 0.741561, - "50" : 0.037547, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json deleted file mode 100644 index b701ea15..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-buffered.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039239, - "timestamp_ms" : 1792039239725, - "time" : "Thu Oct 15 04:40:39 2026", - "jobs" : [ - { - "jobname" : "seqread-buffered", - "groupid" : 0, - "job_start" : 1792039235446, - "error" : 0, - "eta" : 0, - "elapsed" : 5, - "job options" : { - "name" : "seqread-buffered", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", - "rw" : "read", - "bs" : "1M", - "direct" : "0", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "60", - "time_based" : "0" - }, - "read" : { - "io_bytes" : 3996123136, - "io_kbytes" : 3902464, - "bw_bytes" : 936737725, - "bw" : 914782, - "iops" : 893.342710, - "runtime" : 4266, - "total_ios" : 3811, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 139007, - "max" : 19689357, - "mean" : 1117032.838625, - "stddev" : 1439021.300843, - "N" : 3811, - "percentile" : { - "1.000000" : 276480, - "5.000000" : 313344, - "10.000000" : 346112, - "20.000000" : 432128, - "30.000000" : 774144, - "40.000000" : 864256, - "50.000000" : 978944, - "60.000000" : 1089536, - "70.000000" : 1220608, - "80.000000" : 1384448, - "90.000000" : 1531904, - "95.000000" : 1613824, - "99.000000" : 8847360, - "99.500000" : 13303808, - "99.900000" : 18743296, - "99.950000" : 19267584, - "99.990000" : 19791872 - } - }, - "lat_ns" : { - "min" : 139199, - "max" : 19689473, - "mean" : 1117200.393597, - "stddev" : 1439021.295617, - "N" : 3811 - }, - "bw_min" : 679936, - "bw_max" : 1136640, - "bw_agg" : 98.674337, - "bw_mean" : 902656.000000, - "bw_dev" : 179747.109310, - "bw_samples" : 8, - "iops_min" : 664, - "iops_max" : 1110, - "iops_mean" : 881.500000, - "iops_stddev" : 175.534286, - "iops_samples" : 8 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 4265, - "usr_cpu" : 0.257913, - "sys_cpu" : 12.942556, - "ctx" : 3811, - "majf" : 0, - "minf" : 265, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.341118, - "500" : 22.986093, - "750" : 5.484125, - "1000" : 22.644975 - }, - "latency_ms" : { - "2" : 45.945946, - "4" : 1.049593, - "10" : 0.682236, - "20" : 0.865914, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json deleted file mode 100644 index 33e8c57a..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-1M-direct.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039139, - "timestamp_ms" : 1792039139572, - "time" : "Thu Oct 15 04:38:59 2026", - "jobs" : [ - { - "jobname" : "seqread", - "groupid" : 0, - "job_start" : 1792039131591, - "error" : 0, - "eta" : 0, - "elapsed" : 9, - "job options" : { - "name" : "seqread", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", - "rw" : "read", - "bs" : "1M", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "60", - "time_based" : "0" - }, - "read" : { - "io_bytes" : 3996123136, - "io_kbytes" : 3902464, - "bw_bytes" : 501962458, - "bw" : 490197, - "iops" : 478.708705, - "runtime" : 7961, - "total_ios" : 3811, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 1666933, - "max" : 8936381, - "mean" : 2086845.958016, - "stddev" : 478355.966195, - "N" : 3811, - "percentile" : { - "1.000000" : 1728512, - "5.000000" : 1761280, - "10.000000" : 1777664, - "20.000000" : 1810432, - "30.000000" : 1859584, - "40.000000" : 1892352, - "50.000000" : 1941504, - "60.000000" : 1990656, - "70.000000" : 2072576, - "80.000000" : 2211840, - "90.000000" : 2506752, - "95.000000" : 2998272, - "99.000000" : 4177920, - "99.500000" : 4685824, - "99.900000" : 5603328, - "99.950000" : 7241728, - "99.990000" : 8978432 - } - }, - "lat_ns" : { - "min" : 1667036, - "max" : 8937220, - "mean" : 2086990.124902, - "stddev" : 478357.297592, - "N" : 3811 - }, - "bw_min" : 348160, - "bw_max" : 542720, - "bw_agg" : 100.000000, - "bw_mean" : 491383.466667, - "bw_dev" : 62462.641047, - "bw_samples" : 15, - "iops_min" : 340, - "iops_max" : 530, - "iops_mean" : 479.866667, - "iops_stddev" : 60.998673, - "iops_samples" : 15 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 7960, - "usr_cpu" : 0.113065, - "sys_cpu" : 3.994975, - "ctx" : 3813, - "majf" : 0, - "minf" : 268, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_ms" : { - "2" : 60.299134, - "4" : 38.362634, - "10" : 1.338231, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json deleted file mode 100644 index 9fdd891d..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/fio-seqread-256k-qd1-direct.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "fio version" : "fio-3.36", - "timestamp" : 1792039202, - "timestamp_ms" : 1792039202382, - "time" : "Thu Oct 15 04:40:02 2026", - "jobs" : [ - { - "jobname" : "seqread-256k-qd1", - "groupid" : 0, - "job_start" : 1792039172362, - "error" : 0, - "eta" : 0, - "elapsed" : 31, - "job options" : { - "name" : "seqread-256k-qd1", - "filename" : "/mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors", - "rw" : "read", - "bs" : "256k", - "direct" : "1", - "ioengine" : "psync", - "numjobs" : "1", - "runtime" : "30", - "time_based" : "1" - }, - "read" : { - "io_bytes" : 6612844544, - "io_kbytes" : 6457856, - "bw_bytes" : 220420804, - "bw" : 215254, - "iops" : 840.838639, - "runtime" : 30001, - "total_ios" : 25226, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 1035984, - "max" : 4418595, - "mean" : 1187880.284825, - "stddev" : 82024.012557, - "N" : 25226, - "percentile" : { - "1.000000" : 1073152, - "5.000000" : 1089536, - "10.000000" : 1105920, - "20.000000" : 1138688, - "30.000000" : 1155072, - "40.000000" : 1155072, - "50.000000" : 1171456, - "60.000000" : 1187840, - "70.000000" : 1204224, - "80.000000" : 1236992, - "90.000000" : 1269760, - "95.000000" : 1318912, - "99.000000" : 1417216, - "99.500000" : 1466368, - "99.900000" : 1662976, - "99.950000" : 2211840, - "99.990000" : 3424256 - } - }, - "lat_ns" : { - "min" : 1036155, - "max" : 4418682, - "mean" : 1188002.449060, - "stddev" : 82030.606240, - "N" : 25226 - }, - "bw_min" : 202240, - "bw_max" : 223232, - "bw_agg" : 100.000000, - "bw_mean" : 215430.508475, - "bw_dev" : 4740.794515, - "bw_samples" : 59, - "iops_min" : 790, - "iops_max" : 872, - "iops_mean" : 841.525424, - "iops_stddev" : 18.518729, - "iops_samples" : 59 - }, - "write" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "trim" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw_bytes" : 0, - "bw" : 0, - "iops" : 0.000000, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "drop_ios" : 0, - "slat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "clat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - }, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "sync" : { - "total_ios" : 0, - "lat_ns" : { - "min" : 0, - "max" : 0, - "mean" : 0.000000, - "stddev" : 0.000000, - "N" : 0 - } - }, - "job_runtime" : 30000, - "usr_cpu" : 0.253333, - "sys_cpu" : 2.500000, - "ctx" : 25230, - "majf" : 0, - "minf" : 75, - "iodepth_level" : { - "1" : 100.000000, - "2" : 0.000000, - "4" : 0.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_submit" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "iodepth_complete" : { - "0" : 0.000000, - "4" : 100.000000, - "8" : 0.000000, - "16" : 0.000000, - "32" : 0.000000, - "64" : 0.000000, - ">=64" : 0.000000 - }, - "latency_ns" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_us" : { - "2" : 0.000000, - "4" : 0.000000, - "10" : 0.000000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000 - }, - "latency_ms" : { - "2" : 99.944502, - "4" : 0.051534, - "10" : 0.010000, - "20" : 0.000000, - "50" : 0.000000, - "100" : 0.000000, - "250" : 0.000000, - "500" : 0.000000, - "750" : 0.000000, - "1000" : 0.000000, - "2000" : 0.000000, - ">=2000" : 0.000000 - }, - "latency_depth" : 1, - "latency_target" : 0, - "latency_percentile" : 100.000000, - "latency_window" : 0 - } - ] -} diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json b/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json deleted file mode 100644 index 69d2783b..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/monitoring.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "read_bytes_count": { - "timeSeries": [ - { - "metric": { - "labels": { - "file_share": "vol1" - }, - "type": "file.googleapis.com/nfs/server/read_bytes_count" - }, - "resource": { - "type": "filestore_instance", - "labels": { - "location": "us-central1-b", - "project_id": "cdrollouts-sunilarora", - "instance_name": "pvc-b69fc9c0-172e-4423-a7bf-abc5d69a95c1" - } - }, - "metricKind": "GAUGE", - "valueType": "DOUBLE", - "points": [ - { - "interval": { - "startTime": "2026-10-15T04:46:00Z", - "endTime": "2026-10-15T04:46:00Z" - }, - "value": { - "doubleValue": 23058538.966666665 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:45:00Z", - "endTime": "2026-10-15T04:45:00Z" - }, - "value": { - "doubleValue": 19997416.133333333 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:44:00Z", - "endTime": "2026-10-15T04:44:00Z" - }, - "value": { - "doubleValue": 9963643.033333333 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:43:00Z", - "endTime": "2026-10-15T04:43:00Z" - }, - "value": { - "doubleValue": 7131094.833333333 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:42:00Z", - "endTime": "2026-10-15T04:42:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:41:00Z", - "endTime": "2026-10-15T04:41:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:40:00Z", - "endTime": "2026-10-15T04:40:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:39:00Z", - "endTime": "2026-10-15T04:39:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:38:00Z", - "endTime": "2026-10-15T04:38:00Z" - }, - "value": { - "doubleValue": 0 - } - } - ] - } - ], - "unit": "By/s" - }, - "read_ops_count": { - "timeSeries": [ - { - "metric": { - "labels": { - "file_share": "vol1" - }, - "type": "file.googleapis.com/nfs/server/read_ops_count" - }, - "resource": { - "type": "filestore_instance", - "labels": { - "project_id": "cdrollouts-sunilarora", - "location": "us-central1-b", - "instance_name": "pvc-b69fc9c0-172e-4423-a7bf-abc5d69a95c1" - } - }, - "metricKind": "GAUGE", - "valueType": "DOUBLE", - "points": [ - { - "interval": { - "startTime": "2026-10-15T04:45:00Z", - "endTime": "2026-10-15T04:45:00Z" - }, - "value": { - "doubleValue": 210.05 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:44:00Z", - "endTime": "2026-10-15T04:44:00Z" - }, - "value": { - "doubleValue": 63.083333333333336 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:43:00Z", - "endTime": "2026-10-15T04:43:00Z" - }, - "value": { - "doubleValue": 100.35 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:42:00Z", - "endTime": "2026-10-15T04:42:00Z" - }, - "value": { - "doubleValue": 4.55 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:41:00Z", - "endTime": "2026-10-15T04:41:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:40:00Z", - "endTime": "2026-10-15T04:40:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:39:00Z", - "endTime": "2026-10-15T04:39:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:38:00Z", - "endTime": "2026-10-15T04:38:00Z" - }, - "value": { - "doubleValue": 0 - } - } - ] - } - ], - "unit": "{operation}/s" - }, - "write_bytes_count": { - "timeSeries": [ - { - "metric": { - "labels": { - "file_share": "vol1" - }, - "type": "file.googleapis.com/nfs/server/write_bytes_count" - }, - "resource": { - "type": "filestore_instance", - "labels": { - "instance_name": "pvc-b69fc9c0-172e-4423-a7bf-abc5d69a95c1", - "project_id": "cdrollouts-sunilarora", - "location": "us-central1-b" - } - }, - "metricKind": "GAUGE", - "valueType": "DOUBLE", - "points": [ - { - "interval": { - "startTime": "2026-10-15T04:45:00Z", - "endTime": "2026-10-15T04:45:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:44:00Z", - "endTime": "2026-10-15T04:44:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:43:00Z", - "endTime": "2026-10-15T04:43:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:42:00Z", - "endTime": "2026-10-15T04:42:00Z" - }, - "value": { - "doubleValue": 273.06666666666666 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:41:00Z", - "endTime": "2026-10-15T04:41:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:40:00Z", - "endTime": "2026-10-15T04:40:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:39:00Z", - "endTime": "2026-10-15T04:39:00Z" - }, - "value": { - "doubleValue": 0 - } - }, - { - "interval": { - "startTime": "2026-10-15T04:38:00Z", - "endTime": "2026-10-15T04:38:00Z" - }, - "value": { - "doubleValue": 0 - } - } - ] - } - ], - "unit": "By/s" - } -} \ No newline at end of file diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt b/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt deleted file mode 100644 index d17e52aa..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-after.txt +++ /dev/null @@ -1,31 +0,0 @@ -device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 - opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none - age: 150 - caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 - sec: flavor=1,pseudoflavor=1 - events: 52 699 0 0 35 11 819 0 0 3812 0 0 4 0 33 0 0 33 0 0 0 0 0 0 0 0 0 - bytes: 3996123136 0 66593460224 0 70589710968 0 975648 0 - RPC iostats version: 1.1 p/v: 100003/3 (nfs) - xprt: tcp 918 1 2 0 1 118204 118204 0 281157 0 5 68 162921 - per-op statistics - NULL: 1 1 0 44 24 3 1 4 0 - GETATTR: 54 54 0 7744 6048 0 63 65 0 - SETATTR: 0 0 0 0 0 0 0 0 0 - LOOKUP: 19 19 0 3512 4484 0 19 19 0 - ACCESS: 9 9 0 1324 1080 0 9 9 0 - READLINK: 5 5 0 720 980 0 5 5 0 - READ: 118108 118108 0 18424848 70604828792 1928 178931 183807 0 - WRITE: 0 0 0 0 0 0 0 0 0 - CREATE: 0 0 0 0 0 0 0 0 0 - MKDIR: 0 0 0 0 0 0 0 0 0 - SYMLINK: 0 0 0 0 0 0 0 0 0 - MKNOD: 0 0 0 0 0 0 0 0 0 - REMOVE: 0 0 0 0 0 0 0 0 0 - RMDIR: 0 0 0 0 0 0 0 0 0 - RENAME: 0 0 0 0 0 0 0 0 0 - LINK: 0 0 0 0 0 0 0 0 0 - READDIR: 0 0 0 0 0 0 0 0 0 - READDIRPLUS: 2 2 0 336 3124 0 2 2 0 - FSSTAT: 2 2 0 272 168 0 2 2 0 - FSINFO: 2 2 0 272 160 0 2 2 0 - PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt b/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt deleted file mode 100644 index bbfbe3a8..00000000 --- a/runs/2026-10-15_cold_start/fio-l4-lvw8/mountstats-before.txt +++ /dev/null @@ -1,32 +0,0 @@ -lrwxrwxrwx 1 root root 76 Sep 17 22:39 /mnt/shared/open-rl/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218/model-00001-of-00005.safetensors -> ../../blobs/31d6a825ae35f11fb85b195b4c42c146c051e446433125a215336abdf95cbf5f -device 10.80.45.194:/vol1 mounted on /mnt/shared with fstype nfs statvers=1.1 - opts: rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=10.80.45.194,mountvers=3,mountport=2050,mountproto=tcp,local_lock=none - age: 40 - caps: caps=0xf,wtmult=4096,dtsize=1048576,bsize=0,namlen=255 - sec: flavor=1,pseudoflavor=1 - events: 2 102 0 0 2 11 125 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - bytes: 0 0 0 0 0 0 0 0 - RPC iostats version: 1.1 p/v: 100003/3 (nfs) - xprt: tcp 918 1 2 0 0 36 36 0 36 0 2 0 0 - per-op statistics - NULL: 1 1 0 44 24 3 1 4 0 - GETATTR: 4 4 0 552 448 0 6 6 0 - SETATTR: 0 0 0 0 0 0 0 0 0 - LOOKUP: 11 11 0 2008 2596 0 11 11 0 - ACCESS: 8 8 0 1176 960 0 8 8 0 - READLINK: 5 5 0 720 980 0 5 5 0 - READ: 0 0 0 0 0 0 0 0 0 - WRITE: 0 0 0 0 0 0 0 0 0 - CREATE: 0 0 0 0 0 0 0 0 0 - MKDIR: 0 0 0 0 0 0 0 0 0 - SYMLINK: 0 0 0 0 0 0 0 0 0 - MKNOD: 0 0 0 0 0 0 0 0 0 - REMOVE: 0 0 0 0 0 0 0 0 0 - RMDIR: 0 0 0 0 0 0 0 0 0 - RENAME: 0 0 0 0 0 0 0 0 0 - LINK: 0 0 0 0 0 0 0 0 0 - READDIR: 0 0 0 0 0 0 0 0 0 - READDIRPLUS: 2 2 0 336 3124 0 2 2 0 - FSSTAT: 1 1 0 136 84 0 0 0 0 - FSINFO: 2 2 0 272 160 0 2 2 0 - PATHCONF: 1 1 0 136 56 0 1 1 0 diff --git a/runs/2026-10-15_cold_start/fix-image-tag.txt b/runs/2026-10-15_cold_start/fix-image-tag.txt deleted file mode 100644 index a2dbcb0a..00000000 --- a/runs/2026-10-15_cold_start/fix-image-tag.txt +++ /dev/null @@ -1 +0,0 @@ -TAG=coldstart1-dca985a diff --git a/runs/2026-10-15_cold_start/node-labels-before.json b/runs/2026-10-15_cold_start/node-labels-before.json deleted file mode 100644 index ebfdfee8..00000000 --- a/runs/2026-10-15_cold_start/node-labels-before.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "gke-open-rl-dra-h100-sampler-a721f127-0853": { - "cloud.google.com/gke-nodepool": "h100-sampler", - "openrl.io/enabled": "true", - "openrl.io/sampler": "true" - }, - "gke-open-rl-dra-h100-sampler-a721f127-ed3t": { - "cloud.google.com/gke-nodepool": "h100-sampler", - "openrl.io/enabled": "true", - "openrl.io/sampler": "true" - }, - "gke-open-rl-dra-h100-trainer-702095b4-1g3a": { - "cloud.google.com/gke-nodepool": "h100-trainer", - "openrl.io/enabled": "true", - "openrl.io/trainer": "true" - }, - "gke-open-rl-dra-h100-trainer-702095b4-tuqu": { - "cloud.google.com/gke-nodepool": "h100-trainer", - "openrl.io/enabled": "true", - "openrl.io/trainer": "true" - }, - "gke-open-rl-dra-l4-sampler-2x-1ad5171e-bs96": { - "cloud.google.com/gke-nodepool": "l4-sampler-2x", - "openrl.io/enabled": "true", - "openrl.io/sampler": "true" - }, - "gke-open-rl-dra-l4-sampler-2x-ab5fdc9d-2208": { - "cloud.google.com/gke-nodepool": "l4-sampler-2x", - "openrl.io/enabled": "true", - "openrl.io/sampler": "true" - }, - "gke-open-rl-dra-l4-trainer-2x-67d40c4c-sr2c": { - "cloud.google.com/gke-nodepool": "l4-trainer-2x", - "openrl.io/enabled": "true", - "openrl.io/trainer": "true" - }, - "gke-open-rl-dra-l4-trainer-2x-ed601d8b-lvw8": { - "cloud.google.com/gke-nodepool": "l4-trainer-2x", - "openrl.io/enabled": "true", - "openrl.io/trainer": "true" - } -} \ No newline at end of file diff --git a/runs/2026-10-17_demo_baselines/fleet-request-h200.md b/runs/2026-10-17_demo_baselines/fleet-request-h200.md new file mode 100644 index 00000000..a53b4ef2 --- /dev/null +++ b/runs/2026-10-17_demo_baselines/fleet-request-h200.md @@ -0,0 +1,123 @@ +# Fleet request: H200 pool for the fleet demo + +Project `cdrollouts-sunilarora`, cluster `open-rl-dra` (us-central1, GKE 1.35). + +## What we ask for + +- 1 x `a3-ultragpu-8g` (8 x H200 141 GB), zone `us-central1-b`. The H200 has no + smaller shape; 8 GPUs per node is the minimum step. +- Quota: `GPUS-PER-GPU-FAMILY` in `us-central1`: `NVIDIA_H200` to 8 and `NVIDIA_H100` to + at least 10. The H100 limit for new node pools is 0 today: a 1-node a3-highgpu-4g pool + failed with `Quota 'GPUS_PER_GPU_FAMILY' exceeded. Limit: 0.0 in region us-central1` + (2026-10-18 00:33 UTC), while resizing the existing 1-GPU pools from 2 to 3 nodes + worked minutes earlier. The H200 attempt (2026-10-17 23:41 UTC) never reached the + quota check: eight attempts in 35 minutes, all `ZONE_RESOURCE_POOL_EXHAUSTED` in + us-central1-b. +- Also useful: 1 x `a3-highgpu-4g` (4 x H100 on one node, us-central1-a) as the first + multi-GPU node to prove placement across devices before the H200 arrives. +- Capacity: a reservation for 1 machine for the demo window, or approval to use a + flex-start node pool (DWS) if a reservation is not possible. The project has no + reservations today. +- The 4 H100s stay. The L4 pools (8 GPUs) stay until the H200 node is up and has run + one drill. Only then do they go away. Target fleet: 8 H200 + 4 H100 = 12 GPUs. + +## Why H200 and why us-central1-b + +- Same Hopper architecture as the H100s we run today. No new kernels, no new compile + paths. The only new work is one compile-cache warm-up per model. +- us-central1-b is the only zone in the region with `a3-ultragpu-8g`. The Filestore + volume that carries every checkpoint and adapter is also in us-central1-b. +- 2.9 TB of host memory per node. A gemma FFT trainer asks for 91 GiB of host memory, + which limits us to 2 per node on the 1-GPU H100 machines today. + +## Node pool command (run after quota and capacity are confirmed) + +```bash +gcloud container node-pools create h200-8x \ + --cluster open-rl-dra --region us-central1 --node-locations us-central1-b \ + --project cdrollouts-sunilarora \ + --machine-type a3-ultragpu-8g \ + --accelerator type=nvidia-h200-141gb,count=8,gpu-driver-version=disabled \ + --num-nodes 1 \ + --image-type COS_CONTAINERD --disk-type pd-balanced --disk-size 200 \ + --ephemeral-storage-local-ssd count=12 \ + --node-labels openrl.io/enabled=true,openrl.io/trainer=true,openrl.io/sampler=true,nvidia.com/gpu.present=true,gke-no-default-nvidia-gpu-device-plugin=true \ + --node-taints nvidia.com/gpu=present:NoSchedule \ + --scopes default +# With a reservation: add --reservation-affinity specific --reservation +# With flex-start: add --flex-start --enable-autoscaling --min-nodes 0 --max-nodes 1 --num-nodes 0 +``` + +The node carries both role labels; the scheduler lets trainers and samplers share a +node that accepts both. The `gpu-driver-version=disabled` setting matches the existing +pools, where the cluster's own `nvidia-driver-installer` DaemonSet (driver 580) and +the NVIDIA DRA driver v25.8.0 handle the GPUs. + +## After the pool is up + +1. Confirm the DRA ResourceSlice for the node lists 8 H200 devices. +2. Confirm the time-slicer and snapshot-agent DaemonSets run on the node. +3. The pre-pull DaemonSet pulls the worker image on the node (about 3 minutes). +4. Warm the compile cache: gemma-4-e2b and Qwen3-0.6B, LoRA and FFT, on H200. Four + sampler starts. +5. Drill 3: the 12-job drill, then the 24-job drill, with the L4s still in the fleet. +6. If the drills pass, remove the two L4 pools and run the 24-job drill once more. + +## Sizing on 8 H200 + 4 H100 + +The warm sizing asked for about 18 H100 GPUs for 24 jobs. One H200 node is not that, +so the 24-job drill will tell us the real number. What the H200 node changes: 141 GB +per GPU holds more LoRA runtimes and more time-sliced FFT pairs per GPU than an 80 GB +H100, and host memory no longer caps gemma FFT trainers. If 12 GPUs come up short, the +fallback is to keep the L4s for the Qwen3-0.6B LoRA jobs. + +## Quota requests filed 2026-10-19 06:35 UTC (Cloud Quotas API, project cdrollouts-sunilarora) + +All for `GPUS-PER-GPU-FAMILY-per-project-region`, preferred value 16, contact +sunilarora@google.com. The Spot attempt (`h200-spot`, 06:22 UTC) failed on this same quota +(`Limit: 0.0`), so Spot and on-demand share it for these families. + +| Family | Regions | Request IDs | +|---|---|---| +| NVIDIA_H200 | us-central1, us-east4, us-east5, us-east7, us-south1, us-west1, europe-west1, europe-west4, asia-south1, asia-south2 | c3ec12f3, 4c7f0d3c, dbfca2c4, 9bdf2a1a, b2d47b08, 6d7276a1, efd957bb, 07a6ea5e, 60d3071b, aa775f3d | +| NVIDIA_B200 | us-central1, us-east1, us-east4, us-west3, europe-west4, asia-southeast1 | e13d2bc7, 7496ed03, 2748ee31, 5642a24a, 80941b4c, 66973796 | +| NVIDIA_H100 | us-central1 | c640a7da | + +Check status: +`gcloud beta quotas preferences list --project cdrollouts-sunilarora` or the console quota +page. H200 machines (a3-ultragpu-8g) exist in: asia-south1-b, asia-south2-c, europe-west1-b, +europe-west4-a, us-central1-b, us-east4-b, us-east5-a, us-east7-c, us-south1-b, us-west1-c. +B200 (a4-highgpu-8g) in 17 zones including us-east1-b/d, us-east4-b, us-west3-b/c, +europe-west4-b, asia-southeast1-b. + +## A second cell in another region + +A GKE cluster's node pools must be in the cluster's region, so capacity in another region +means a second OpenRL cell there: a regional cluster, a Filestore instance, the bundle +with the dev images, and the client Jobs, all in that region. One control plane across +regions is not possible with GKE node pools; a GKE Fleet groups the clusters for one view +but each keeps its own control plane. + +## Result of the CLI requests (checked 06:50 UTC, about 15 minutes after filing) + +Every one of the 17 requests came back `Quota request denied` by the automatic review, in +every region, for H200, B200 and the H100 family alike. The one approved entry in the +project's history is an older request without a family dimension, granted to 10. So the +self-service path does not open these families for this project. The request has to go +through the account team or a support case, which is a person, not the API. + +What to send them: project `cdrollouts-sunilarora`, quota `GPUS-PER-GPU-FAMILY-per-project-region`, +family `NVIDIA_H200` (and `NVIDIA_B200`), value 16, in us-central1 plus two or three of +us-east4, us-east5, us-south1, us-west1, and the Spot use for drills with a reservation for +the demo date. The request IDs above are the evidence of the automatic denial. + +## Cell dry run, us-east4 (2026-10-19 06:40-07:10 UTC) + +`scripts/cell_up.sh cell-use4 us-east4 us-east4-b` created the regional cluster and the +Spot a3-ultragpu-8g pool; the pool failed on `GPUS_PER_GPU_FAMILY ... Limit: 0.0 in region +us-east4`, as expected, and the script reported it every 20 s until stopped. The cluster +was then deleted with `scripts/cell_down.sh`; no Filestore instance was left behind. The +bundle apply on a fresh cluster is still unverified (my check ran from zsh, where the +script's `$K` variable does not word-split; the script itself is bash). Verify it the +first time a cell gets a GPU node. + diff --git a/runs/2026-10-17_demo_baselines/notes.md b/runs/2026-10-17_demo_baselines/notes.md new file mode 100644 index 00000000..1dc9d7e5 --- /dev/null +++ b/runs/2026-10-17_demo_baselines/notes.md @@ -0,0 +1,530 @@ +# Fleet demo: measurements and drills (2026-10-17 to 2026-10-18) + +## The goal + +Run 20 to 25 GSM8K GRPO jobs at the same time. Each job uses 8 prompts, 8 samples per +prompt, and 512 max tokens. Half of the jobs use LoRA and half use full fine-tuning +(FFT). The models are gemma-4-e2b and Qwen3-0.6B. The fleet mixes H100 and L4 GPUs. +The target: 75% of the jobs finish 2 or 3 steps within 2 or 3 minutes of kickoff. + +Clients run as in-cluster Jobs from `scripts/demo_jobs.py`. Timings come from each +recipe's own `metrics.jsonl`, read by `scripts/demo_report.py --per-step`. + +Cluster: open-rl-dra on GKE. At the start: 4 H100 (2 trainer nodes, 2 sampler nodes) +and 8 L4 (2 trainer nodes and 2 sampler nodes, 2 GPUs each). Storage: Filestore Basic +HDD. Every run below started its workers cold unless the text says otherwise. + +## Summary + +Scorecard, jobs that finished 3 steps within 180 s: + +| Drill | Fleet | Setting | Result | +|---|---|---|---| +| 1 | 4 H100 + 8 L4 | binpack, no fixes | 0 of 12 | +| 2 | 4 H100 + 8 L4 | binpack, all cold-start fixes | 0 of 12, 1 failed | +| 3 | 6 H100 | binpack | 3 of 12 | +| 4 | 6 H100 | binpack, seat limit 2 | **7 of 12** | +| 5 | 6 H100 + 8 L4 | binpack, seat limit 2 | 0 of 12 | +| 6 | 6 H100 | binpack, seat limit 2, 24 jobs | 10 of 24 | +| 7 | 6 H100 | binpack, seat limit 2, LoRA runtimes warm | **8 of 12** | + +What we learned, in order of weight: + +1. **A cold start cannot meet the window; a warm runtime can.** A sampler needs 60 to + 130 s to start on an H100 after every fix. A shared LoRA runtime with three jobs + waiting needs 165 s for its first step. Drill 7 ran the same jobs onto runtimes that + were already up: three gemma LoRA jobs finished three steps in about 90 s instead of + 245 s, and the drill went from 7 to 8 of 12 with the gemma LoRA group moving from the + slowest to the fastest. +2. **The seat limit works.** With at most two workers per shared claim, no job runs more + than two-way shared. The 12-job drill on six H100s went from 3 to 7 inside the window. +3. **The L4s hurt as long as binpack may choose them.** Binpack takes the smallest device + that fits. The LoRA runtimes land on L4s and run three times slower. The same 12 jobs + went from 7 to 0 when the L4s came back. A device-tier preference is the lever. +4. **gemma FFT cannot run on an L4.** Its weight update needs 11 GiB of transient memory. + Fixed: the sampler wakes the KV cache after the update, and the claim now says 24 GiB, + so the scheduler puts it on an H100 by size. +5. **Training was slow for a bad reason.** The cluster never set a token budget, so the + trainer ran one sequence per pass. The budget is automatic now. +6. **Storage grows under load.** Finished FFT jobs leave 2 to 10 GB each behind. Clean + between drills. Under a burst, adapter saves on Basic HDD took up to 88 s. + +## Part 1: one job alone + +Images `coldstart1-dca985a` (prefetch fix in), client `demo1-18409bd`. No token budget set. + +| Job | GPU | Step 1, includes worker start | Warm step: sampling + train + save = total | 3 steps done | +|---|---|---|---|---| +| LoRA Qwen3-0.6B | L4 | 187 s (sampler start 150) | 17 + 30 + 1 = 49 s | 284 s | +| FFT Qwen3-0.6B | L4 | 147 s | 18-20 + 21 + 2 = 41-44 s | 232 s | +| LoRA Qwen3-0.6B | H100 | 130 s (sampler start 106) | 5 + 20 + 1 = 26 s | 183 s | +| FFT Qwen3-0.6B | H100 | 194 s (sampler start 170) | 9 + 18-23 + 2 = 29-35 s | 259 s | +| LoRA gemma-4-e2b | H100 | 288 s (sampler start 240) | 9 + 31-35 + 2-4 = 42-49 s | 378 s | +| FFT gemma-4-e2b | H100 | 247 s (sampler start 175) | 15-18 + 31-35 + 12-15 = 58-67 s | 371 s | + +The client pod itself: 26 s to pull its image on a node without the tag, then 11 s from +container start to the first API request. + +What the numbers say: + +- **The worker start dominates the first step.** The sampler takes 106 to 240 s of it: + vLLM load, compile, and graph capture. +- **Training is the largest warm-step cost, and the H100 barely helps.** A 0.6B model took + 20 s for 64 sequences. The cause: the deployment did not set + `OPEN_RL_TRAIN_TOKEN_BUDGET`, so `make_training_batches` ran one sequence per + forward-backward pass, 64 passes per step. The e2e harness sets 65536 locally. The + cluster never did. +- **Sampling is fine.** 5 to 10 s on H100 and 17 to 20 s on L4 for 64 completions. +- **FFT weight sync is the storage cost per step.** `time/save_checkpoint` is 12 to 15 s + for gemma on Basic HDD and 2 s for 0.6B. It grows with the fleet. + +### Token budget 65536, set on the api-server and forwarded to workers + +| Job | GPU | Step 1 | Warm step: sampling + train + save = total | 3 steps done | Train before -> after | +|---|---|---|---|---|---| +| LoRA Qwen3-0.6B | H100 | 110 s | 5 + 3-5 + 1 = 9-11 s | 130 s | 20 -> 4 s | +| FFT Qwen3-0.6B | H100 | 95 s | 10 + 6-8 + 2 = 19-20 s | 134 s | 18-23 -> 7 s | + +One environment variable. A warm 0.6B job now does 3 steps in about 30 s on an H100. + +gemma-4-e2b failed with 65536. Both gemma trainers got `CUDA out of memory. Tried to +allocate 21.94 GiB` on an 80 GB H100. The client got a 400 and the Job failed within 5 +minutes, so the error path works. The cause: gemma has a 262k vocabulary, so the logits +of a 65k-token batch are about 22 GiB in bf16. The budget must depend on the model's +vocabulary and the device. It cannot be one number for the fleet. + +### Token budget 8192, gemma-4-e2b on H100 + +| Job | Step 1 | Warm step: sampling + train + save = total | Train before -> after | +|---|---|---|---| +| LoRA gemma-4-e2b | 257 s | 9-11 + 8-10 + 2 = 19-23 s | 31-35 -> 8-10 s | +| FFT gemma-4-e2b | 229 s | 15-18 + 12-27 + 11-14 = 44-54 s | 31-35 -> 12-27 s | + +gemma FFT is now bound by sampling (15 to 18 s) and by the weight sync write on Basic HDD +(11 to 14 s). The FFT sampler goes through the time-slicer even when it is alone. + +Code change: the trainer now derives the budget from the model's vocabulary and the free +device memory (`train_token_budget`). It caps any explicit number the same way. No +deployment has to pick a number per fleet. + +### Token budget 8192, Qwen3-0.6B on L4 + +| Job | Step 1 | Warm step: sampling + train + save = total | Train before -> after | +|---|---|---|---| +| LoRA Qwen3-0.6B | 176 s | 17 + 18 + 1 = 36 s | 30 -> 18 s | +| FFT Qwen3-0.6B | 145 s | 19 + 15 + 2 = 36 s | 21 -> 15 s | + +On an L4 the step splits evenly between sampling and training. Both are compute-bound +there. A warm L4 job does 3 steps in about 110 s. + +### Warm steps after the budget, one job alone + +| Job | H100 | L4 | +|---|---|---| +| LoRA Qwen3-0.6B | 9-11 s | 36 s | +| FFT Qwen3-0.6B | 19-20 s | 36 s | +| LoRA gemma-4-e2b | 19-23 s | not run | +| FFT gemma-4-e2b | 44-54 s | not run | + +The automatic budget (image `demo2-22fa006`, no budget env) gave the same numbers: +LoRA Qwen3-0.6B on L4 had warm steps of 34 to 36 s with 17 s of training. LoRA gemma on +L4 had warm steps of 77 to 81 s and a sampler start of 190 s, of which 122 s was +torch.compile. gemma does not belong on an L4 for this demo. + +## Part 2: jobs that share a GPU pair + +All on H100, budget 8192, workers started cold. + +### Three Qwen3-0.6B LoRA jobs on one runtime + +| Job | Step 1 | Warm steps | 3 steps done | +|---|---|---|---| +| lora 01 | 122 s | 20.6, 21.1 s | 164 s | +| lora 02 | 126 s | 20.5, 18.8 s | 165 s | +| lora 03 | 123 s | 20.5, 21.1 s | 165 s | + +A shared LoRA runtime serves three jobs at about 20 s per step each (9 to 11 s alone). +Training serializes and sampling overlaps. All three finished inside 180 s even with the +cold start. Warm, 3 steps take about 60 s. + +### Three Qwen3-0.6B FFT jobs time-slicing one pair + +Binpack put all three pairs on the same two H100s. + +| Job | Step 1 | Warm steps | 3 steps done | +|---|---|---|---| +| fft 01 | 246 s | 26.4, 37.2 s | 310 s | +| fft 02 | 239 s | 31.9, 32.4 s | 303 s | +| fft 03 | 254 s | 25.7, 37.4 s | 317 s | + +Time-sliced FFT runs at 26 to 37 s per step each (19 to 20 s alone). Three jobs' warm +steps take about 95 s. The cold start tripled to 220 s because three vLLM samplers +initialize in turn on one GPU. + +### gemma-4-e2b + +Three LoRA jobs on one runtime: warm steps of 34 to 42 s per job (19 to 23 s alone), +about 110 s for 3 warm steps, and a cold start of 240 to 260 s. + +Three FFT jobs: only two could be placed. The third trainer waited with +`WaitingForCapacity ... 1 Insufficient memory` until a job finished. A gemma FFT trainer +pod asks for 91 GiB of host memory for the parked copies that the time-slicer and CPU +offload need. Two of them fill the trainer node's RAM. The two that ran took 44 to 66 s +per step (44 to 54 s alone), about 150 to 180 s for 3 warm steps. That is at the 3-minute +line. gemma FFT needs its own capacity plan: 2 per pair. + +### Fleet sizing from these numbers, runtimes warm + +| Job type | Jobs per GPU pair | 3 warm steps per job | Limit | +|---|---|---|---| +| LoRA Qwen3-0.6B | 3 | about 60 s | trainer serializes; 4 would still fit | +| FFT Qwen3-0.6B | 3 | about 95 s | time-slice switches | +| LoRA gemma-4-e2b | 3 | about 110 s | trainer serializes | +| FFT gemma-4-e2b | 2 | 150-180 s | 91 GiB host memory per trainer | + +For 24 jobs split evenly over the four types, that is 2 + 2 + 2 + 3 = 9 H100 pairs, or +18 H100 GPUs. The gemma FFT jobs decide whether 75% make it. L4s can carry the 0.6B LoRA +share: one job per L4 pair does 3 steps in about 110 s. + +Everything in this table assumes warm runtimes. Cold starts measured on this day: 100 to +130 s for a 0.6B sampler, 240 to 260 s for a gemma sampler, and 220 to 330 s when three +samplers start on one time-sliced GPU. + +## Part 3: making the cold start fast + +### Pre-pull DaemonSet `open-rl-worker-prepull`, first rollout 20:36 UTC + +Eight nodes. Three already had the tag. Four H100 nodes pulled it in 1.6 to 2.5 s: the +new tag shares every layer but the last with an image already on the node. One L4 node +had none of the dev layers and pulled the full image. A tag bump costs seconds when the +big layers are present and 90 to 170 s when they are not. For the demo: build the demo +image on the same base the nodes hold, or pre-pull once and keep it. + +### vLLM compile cache on the node + +hostPath `/var/lib/openrl/vllm-cache`, api-server `demo3-3f5d1bb`. The same LoRA +Qwen3-0.6B sampler started three times on L4 nodes. + +| Run | Node | Cache state | torch.compile | engine init | Step 1 (sampling part) | +|---|---|---|---|---|---| +| k1 | 2208 | empty | 55.7 s | 88 s | 176 s (151) | +| k2 | bs96 | empty, other node | 56.0 s | 88 s | 175 s (151) | +| k3 | bs96 | warm from k2 | **0.9 s** | **18 s** | **94 s (70)** | + +The cache is keyed by model, GPU type, and engine config. The first sampler of a +combination on a node pays the compile once. Every later one loads the AOT artifact +("Directly load AOT compilation"). Graph capture is not cached and still runs. + +### vLLM compile cache on the shared volume + +`OPEN_RL_VLLM_CACHE_DIR=/mnt/shared/open-rl/vllm-cache`, api-server `demo4-459e718`. + +| Run | Node | Shared cache | torch.compile | engine init | Step 1 (sampling part) | +|---|---|---|---|---|---| +| sc-s1 | bs96 | empty, written here | 67.1 s | 106 s | 195 s (170) | +| sc-s2 | 2208, never ran the model | warm from s1 | **1.1 s** | **20 s** | **95 s (71)** | + +The shared directory gives the same gain as the hostPath, from any node, and it survives +node recreation. One model on one GPU type is 61 MB in 913 files. The first write costs +about 10 s more than local disk. The bundle now defaults to the shared directory. The +hostPath stays as the option for installs without a shared volume. + +### Where the sampler's start time goes + +A probe inside the worker image on an L4 node, no GPU work: + +- `uv run python -c pass`: 0.23 s. The uv wrapper is not the cost. +- `import server.vllm_sampler`: 11.0 s (vllm 4.5, vllm.config 3.1, torch 1.8, engine 1.0). +- `import server.training_requests_processor`: 7.3 s. + +vLLM V1 runs the engine core as a second process that imports everything again. That is +the 18 to 28 s of "imports until engine init" in the timelines. + +### Sampler start A/B, gemma-4-e2b LoRA on H100 node 0853 + +Images `demo5-8736ff0`, shared compile cache. + +| Variant | torch.compile | graph capture, two passes | engine init | first step (sampling part) | warm step | +|---|---|---|---|---|---| +| vLLM default capture list | 102 s (cold entry) | 29 + 24 = 53 s | 199 s | 301 s (285) | 23 s | +| power-of-two capture sizes | 103 s (new entry: the sizes are part of the key) | 19 + 11 = 30 s | 175 s | 272 s (257) | 21 s | +| pow2 + in-process engine | 2.4 s (warm) | 12 + 11 = 23 s | 59 s | 151 s (136) | 24 s | +| pow2, warm, engine as a subprocess | 1.4 s | 12 + 10 = 22 s | **57 s** | **147 s (132)** | 24 s | + +Reading it: + +- Power-of-two capture saves about 23 s per start on an H100 for gemma. +- The in-process engine changes nothing (57 vs 59 s). Leave vLLM's default. +- gemma's first compile on an H100 is 100 s. That is the one-time cost a warm-up pays per + model, per GPU type, per fine-tuning type. Change the capture list before warming. +- Warm steps were the same in all variants. None of the knobs cost throughput. + +What this delivered for a gemma sampler on an H100: engine init 199 -> 57 s and first +step 301 -> 147 s. The shared cache gave 100 s and power-of-two capture gave 23 s. The +57 s that remain: weights 13 s, compile 1.4 s, capture 22 s, profiling and KV cache +about 20 s. About 40 s of pod start, imports, and config come before the engine line. +About 20 s sit between "ready" and the client's first samples. + +## Part 4: the drills + +### Drill 1, 21:00 UTC: 12 jobs, cold, binpack + +Mix for every 12-job drill: gemma LoRA x3, gemma FFT x2, 0.6B LoRA x3, 0.6B FFT x4. +Pre-pull and the compile cache were in place. Result: **0 of 12**. + +| Group | Where the scheduler put it | 3 steps done | +|---|---|---| +| 0.6B LoRA x3, one runtime | L4 | 200-223 s | +| 0.6B FFT x4 | 2 on H100 pairs, 2 on L4 | 312-434 s | +| gemma FFT x2 | H100, samplers packed 3 to a GPU | 465-469 s | +| gemma LoRA x3, one runtime | **L4** | about 630 s | + +Two causes: + +1. **Cold start.** Every runtime was built after kickoff. The fastest group's first step + was 88 s. The gemma sampler on an L4 took 192 s to initialize. +2. **Placement by fit, not by speed.** The scheduler packs by memory. A 10 GB gemma + sampler fits an L4, so the gemma LoRA runtime went there at 80 s per step while two + 0.6B FFT pairs took H100s. A Workload asks for a memory size and a role, nothing else. + +Storage after the drill: 17%. `sampler_full` grew from 34 to 66 GB in one day. Finished +FFT jobs leave their sampler weights, checkpoints, and adapters behind: 1.8 GB for 0.6B +and about 10 GB for gemma. The demo needs a cleanup step between drills and a retention +rule. + +### Drill 2, 22:43 UTC: the same 12 jobs, cold, every cold-start fix in + +The worker image was on every node. The compile cache was warm for the LoRA samplers. +CUDA graphs used the power-of-two list. The token budget was automatic. +Result: **0 of 12**, one failed. The fastest group needed 219 s. + +| Group | Where | First step | Warm steps | 3 steps done | +|---|---|---|---|---| +| 0.6B LoRA x3, one runtime | L4 | 105-120 s | 56-61 s | 219-240 s | +| 0.6B FFT x4 | 2 pairs on L4, 1 on H100 | 187-290 s | 40-53 s | 275-376 s, one failed | +| gemma FFT x2 | H100, samplers 3 to a GPU | 422-429 s | 43-58 s | 510-539 s | +| gemma LoRA x3, one runtime | **L4**, same nodes as the 0.6B runtime | 290 s | **164-175 s** | 630 s | + +Four things happened only because the jobs started together: + +1. **Placement.** Both LoRA runtimes went to the same two L4 nodes. The H100s took the + FFT pairs. On a shared L4 one gemma LoRA step takes 165 s. +2. **Storage contention.** Sixteen workers started on one Basic HDD volume at once. An + adapter save takes 1 to 2 s alone. In the burst it took 17 s for 0.6B and 47 to 88 s + for gemma, on every step. +3. **The compile cache key includes the engine config.** An FFT sampler runs without LoRA + and with sleep mode, so its key differs from a LoRA sampler's key. The warm-up had + warmed only the LoRA keys. Each gemma FFT sampler compiled for 100 s, three in turn + on one GPU. That is the 420 s first step. A warm-up must cover model x GPU type x + fine-tuning type. +4. **One client failed on NFS.** `OSError: [Errno 116] Stale file handle` from `filelock` + inside `datasets.load_dataset`. Twelve clients opened one Hugging Face cache on the + shared volume at once. Fixed later: each client seeds its own cache on an emptyDir. + +### Drill 3, 2026-10-18 00:11 UTC: 6 H100 only + +Changes since drill 2: both H100 pools grew from 2 to 3 nodes. The four L4 nodes carry +`openrl.io/trainer=false` and `openrl.io/sampler=false`. The compile cache was warm for +all eight keys on H100. Each client seeds its own Hugging Face cache. +Result: **3 of 12**. All 12 succeeded. + +| Group | First step | Warm steps | 3 steps done | +|---|---|---|---| +| 0.6B LoRA x3, one runtime | 59-63 s | 20 s | 100-106 s | +| gemma LoRA x3, one runtime | 166-172 s | 34-51 s | 258-261 s | +| gemma FFT x2 | 167, 252 s | 98-148 s | 294, 517 s | +| 0.6B FFT x4 | 235-260 s | 89-148 s | 413-498 s | + +Where the 16 workers went, from the Scheduled events: + +| GPU | Workers | +|---|---| +| sampler 0853 | gemma LoRA sampler, alone | +| sampler h9sb | 0.6B LoRA sampler, alone | +| sampler ed3t | **all six FFT samplers** | +| trainer 1g3a | gemma LoRA trainer, alone | +| trainer 7zrs | 0.6B LoRA trainer + 1 FFT trainer | +| trainer tuqu | **five FFT trainers** | + +The finding: the scheduler packs without a limit. Six FFT samplers time-sliced one H100 +while two sampler GPUs each held one LoRA runtime. FFT sampling took 110 to 130 s per +step. A second finding: three jobs on one LoRA runtime serialize the first step, 133 s +of sampling against about 60 s alone. + +### Drill 4, 01:51 UTC: 6 H100, seat limit 2 + +Workers `demo7-87417c2`, scheduler `demo8-943eb38` with `OPEN_RL_MAX_SEATS_PER_CLAIM=2`. +Result: **7 of 12**. All 12 succeeded. + +| Group | 3 steps done | Drill 3 | +|---|---|---| +| 0.6B LoRA x3 | 95-97 s | 100-106 s | +| 0.6B FFT x4 | 112-180 s | 413-498 s | +| gemma LoRA x3 | 245-246 s | 258-261 s | +| gemma FFT x2 | 280-287 s | 294, 517 s | + +At +2 min: both LoRA runtimes held H100 pairs exclusively. Two FFT pairs shared one H100 +pair with two seats each. The other four FFT trainers were Pending on standing claims of +their own, with the reason `WaitingForCapacity: ... shared claims hold at most 2 +workers`. They took seats as the first jobs finished. No worker ran more than two-way +shared. + +What is left is GPU count. Two exclusive LoRA runtimes hold 4 of the 6 GPUs for 6 jobs, +so 6 FFT jobs queue behind one pair. + +### Drill 5, 02:02 UTC: 6 H100 + 8 L4, seat limit 2 + +Result: **0 of 12**. All 12 succeeded. Every worker ran at once, no Pending. + +| Group | Where | 3 steps done | Drill 4, H100 only | +|---|---|---|---| +| 0.6B LoRA x3 | runtime on L4 | 232-270 s | 95-97 s | +| 0.6B FFT x4 | two L4 pairs, 2 seats each | 195-237 s | 112-180 s | +| gemma LoRA x3 | trainer on L4, sampler on H100 | 559-568 s | 245 s | +| gemma FFT x2 | H100, 2 seats | 277-303 s | 280-287 s | + +Binpack takes the smallest device that fits. The LoRA runtimes and the Qwen FFT pairs +went to the L4s while H100 GPUs sat idle. Only gemma FFT stayed on H100, because its +claim no longer fits an L4. The seat limit held and changed nothing here. Adding eight +L4 GPUs made the drill slower than the six H100s alone. The lever is a device-tier +preference for the LoRA runtimes. It is on hold. + +### Drill 6, 02:23 UTC: 24 jobs on 6 H100, seat limit 2 + +Six jobs of each type. Result: **10 of 24**. All 24 succeeded. The last job finished at ++480 s. + +| Group | 3 steps done | Note | +|---|---|---| +| 0.6B LoRA, one runtime | 122-149 s | six jobs on one runtime, all inside the window | +| 0.6B FFT | 106-215 s | queued two at a time behind one pair, 4 of 6 inside | +| gemma FFT | 196-326 s | same queue, after the Qwen FFT jobs | +| gemma LoRA, one runtime | 398-480 s | six jobs on one runtime, 75-80 s of sampling per step | + +At +3 min: 9 workers running, 8 FFT trainers Pending with the limit in the reason. The +queue drained in about 5 minutes. No claim ever held more than two seats. Storage went +from 13% to 16% during the run. + +Reading: six H100s do 24 jobs in 8 minutes with every job bounded, and the Qwen half +mostly inside 3 minutes. The gemma LoRA runtime needs more: a warm runtime, a second +sampler, or fewer jobs per runtime. + +## Part 5: gemma FFT on an L4, the weight-update bug + +### The probe, 00:42 UTC + +L4s back in, one gemma LoRA job and one gemma FFT job alone on an idle fleet. Binpack +put the gemma LoRA trainer and sampler on L4s, and the gemma FFT sampler on an L4 with +its trainer on an H100. + +| Job | First step | Warm steps | Result | +|---|---|---|---| +| gemma LoRA alone on L4 | 202 s | 76 s | 3 steps at 354 s | +| gemma FFT, sampler on L4 | 134 s | none | **failed at the first weight update** | + +The sampler died with `CUDA out of memory. Tried to allocate 4.38 GiB` and 1.86 GiB free +of 22.03 GiB. A gemma step on an L4 is about three times the H100 step even alone. + +### Where the memory goes + +The sampler applies a sparse delta through vLLM's checkpoint patch loader. For each +patched checkpoint tensor the loader allocates a dense NaN-filled copy in bf16. Then its +patched `Tensor.copy_` computes `torch.where(torch.isnan(src), dst, src)`: a bool mask +and a second dense result. gemma-4-e2b's per-layer input embedding has 2.35 billion +elements. Its update needs 4.38 + 2.19 + 4.38 = 10.95 GiB at once, on top of 9.0 GiB of +weights and 1.4 GiB of CUDA graphs. + +| vLLM share on the L4 | Free at update time | Result | +|---|---|---| +| 0.90, the default | 1.9 GiB | OOM at the 4.38 GiB dense copy | +| 0.68, headroom = 1x largest tensor + 0.5 GiB | about 5.7 GiB | OOM at the 2.19 GiB mask | +| 0.45, forced | about 10.7 GiB | 3 steps in 284 s, steps of 70 s, KV cache 0.38 GiB | + +### The fix, commits d5361ed and 87417c2, verified at 01:36 UTC with images demo7-87417c2 + +- The sampler wakes only the weights when it takes the GPU. It applies the update, then + wakes the KV cache. This is vLLM's documented order for RL weight updates. +- The runtime headroom is 2.5 times the largest checkpoint tensor plus 1 GiB, read from + the safetensors headers. vLLM's share never drops below the resident weights plus 1 GiB. + `OPEN_RL_WEIGHT_UPDATE_HEADROOM_GIB` overrides it. +- The estimator adds the same room to the FFT sampler's placement claim. gemma FFT's claim + went from 11.7 to 23.6 GiB. It no longer fits a 24 GB device. + +Verification on the idle full fleet: the scheduler put the gemma FFT sampler on an H100 +with no hint. The log shows wake weights, update, wake KV cache on every step. The job +finished 3 steps in 225 s with warm steps of 39 to 44 s, the single-job H100 number. The +KV cache on the H100 is 48 GiB. + +## Part 6: other probes + +### gemma LoRA x3 with `openrl.sampler_replicas=2`, 02:15 UTC, 6 H100 + +3 steps in 233 to 235 s, against 245 s with one replica in drill 4. Warm steps fell from +about 82 s to 34 to 35 s, so two samplers halve the shared runtime's step. The first step +did not move: 164 to 166 s, of which 128 to 134 s is the sampler's own start with three +jobs waiting on it. + +So a shared LoRA runtime on a cold start has a floor of about 165 s plus 35 s per further +step. Three steps inside 180 s needs the runtime already warm. A warm runtime would do the +same three jobs in about 105 s. That is the number for the warm-standby discussion. A +second replica costs a sampler GPU per runtime. On six H100s that starves the FFT +samplers, so it is a setting for the H200 fleet. + +### The H200 pool, 23:41 UTC + +`gcloud container node-pools create h200-8x` with one a3-ultragpu-8g in us-central1-b +needed two config fixes: a Hyperdisk Balanced boot disk and the fixed 32 local SSDs. It +then failed on capacity: eight attempts in 35 minutes, all `ZONE_RESOURCE_POOL_EXHAUSTED`. +A later a3-highgpu-4g pool failed in seconds on quota: `GPUS_PER_GPU_FAMILY ... Limit: 0.0 +in region us-central1`. Both pools were deleted. The request text is in +`fleet-request-h200.md` in this directory. + +## Where things stand, 02:40 UTC + +Cluster: api-server and workers on `demo7-87417c2`, scheduler on `demo8-943eb38` with +`OPEN_RL_MAX_SEATS_PER_CLAIM=2`. The L4 nodes are labeled out of the drill. No workers +are running. Storage is at 16%. + +Branch `feat/cold-start-measure`, unpushed: + +``` +fcf8b8d scripts: let demo Jobs carry openrl.* session tags +943eb38 scheduler: cap the seats a shared claim may hold +87417c2 sampler: wake the KV cache after the weight update, and size the update's room at its peak +ed0f629 scripts: give each demo client its own Hugging Face cache +d5361ed sampler: keep room for the weight update's dense staging copy +``` + +Open decisions: + +- A device-tier preference for the LoRA runtimes, so binpack does not choose an L4 for + gemma. On hold. +- Warm standby runtimes, so the first step does not pay the sampler start. On hold. +- A per-tier seat limit, one number for 24 GB devices and another for 80 GB. Later. +- The fleet request: quota for the H100 and H200 families, and capacity for one H200 node. + +### Drill 7, 2026-10-19 07:18 UTC: 6 H100, seat limit 2, LoRA runtimes warm + +Two one-step jobs brought the gemma and Qwen LoRA runtimes up first. The 12 jobs launched +five seconds after those finished. Result: **8 of 12**. All 12 succeeded. + +| Group | 3 steps done | Drill 4, cold runtimes | +|---|---|---| +| gemma LoRA x3 | 88-97 s | 245-246 s | +| 0.6B LoRA x3 | 109-121 s | 95-97 s | +| 0.6B FFT x4 | 134-211 s | 112-180 s | +| gemma FFT x2 | 235-274 s | 280-287 s | + +The gemma LoRA group is the whole story: first step 21 to 31 s instead of 170 s. The Qwen +LoRA group was slower than in drill 4 because its sampler was reaped in the five seconds +between the warm-up and the kickoff (the API server removes a runtime about two minutes +after its last session ends, and the Qwen warm-up job had finished first), so its jobs +paid a sampler start again. A real standby needs a keep-alive, not timing luck. The FFT +groups are unchanged: they still queue behind one H100 pair. + +Added this morning: a Reason column on `kubectl get workloads` (commit aba8144, CRD +applied), `scripts/demo_rehearsal.sh` (idle check, volume trim, optional warm-up, +kickoff, watch), and the cell scripts for other regions (commit fc1ccbb). Quota: all 17 +self-service requests for H200, B200 and H100 families were auto-denied within seconds; +see fleet-request-h200.md. +