From 4702d1ff005ee42a1b0e1067857bc10e6cf60866 Mon Sep 17 00:00:00 2001 From: Marshall Date: Fri, 6 Feb 2026 16:42:23 -0500 Subject: [PATCH 001/382] Update app.py --- app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app.py b/app.py index 07069c25..b61e4cd7 100644 --- a/app.py +++ b/app.py @@ -13,9 +13,6 @@ from flask import Flask, send_from_directory, request, jsonify, session from collections import deque -os.environ.pop("DATABRICKS_CLIENT_ID", None) -os.environ.pop("DATABRICKS_CLIENT_SECRET", None) - # Session timeout configuration SESSION_TIMEOUT_SECONDS = 60 # No poll for 60s = dead session CLEANUP_INTERVAL_SECONDS = 30 # How often to check for stale sessions From 1f57fbd88826f531f699a99c2a7733af5acac842 Mon Sep 17 00:00:00 2001 From: Marshall Date: Fri, 6 Feb 2026 16:44:04 -0500 Subject: [PATCH 002/382] Apply suggestion from @mpkrass7 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 20b286c8..f44f2e63 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ databricks apps deploy xterm-terminal --source-code-path /Workspace/Users/` with your Databricks username (e.g., `user@example.com`). -Once the app is deployed. You'll need to add the 'DATABRICKS_TOKEN' secret to your Databricks workspace and reference it in the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources). +Once the app is deployed, create a secret with your PAT in your Databricks Workspace. In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources), add the secret aliased as DATABRICKS_TOKEN. ### Automatic Git Configuration From 647d7fa4d02842ff27600d1f12259912ca89d190 Mon Sep 17 00:00:00 2001 From: "sathish.gangichetty" Date: Thu, 19 Feb 2026 23:17:40 -0500 Subject: [PATCH 003/382] fix: graceful terminal session cleanup on exit and tab close - Replace DELETE /api/session with POST /api/session/close that properly kills the shell process via SIGHUP/SIGKILL instead of just closing fd - Detect process exit in reader thread via select exceptional conditions and waitpid fallback, marking session as exited - Frontend polls detect exited flag, show message, and clean up - beforeunload handler now hits correct POST endpoint via sendBeacon - Update default model to databricks-claude-opus-4-6 Co-Authored-By: Claude Opus 4.6 --- README.md | 8 ++++--- app.py | 61 ++++++++++++++++++++++++++++++++++------------- app.yaml | 2 +- static/index.html | 27 +++++++++++++++++---- 4 files changed, 73 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index f44f2e63..71d8934c 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,15 @@ Just use it all on Databricks, from the browser. Wired up to model serving endpo ✅ **Real-time I/O** - Responsive terminal with polling-based communication +✅ **Graceful Session Cleanup** - Shell processes are properly terminated on exit, tab close, or timeout + ✅ **Terminal Resizing** - Dynamic resize support for responsive layouts ✅ **Databricks Workspace Integration** - Auto-sync projects to Databricks Workspace on git commits ✅ **Claude Code CLI** - Pre-configured to use Databricks hosted models as the API endpoint -✅ **Configurable Model** - Switch between Claude models via `app.yaml` (default: `databricks-claude-sonnet-4-5`) +✅ **Configurable Model** - Switch between Claude models via `app.yaml` (default: `databricks-claude-opus-4-6`) ✅ **Micro Editor** - Ships with [micro](https://micro-editor.github.io/), a modern terminal-based text editor @@ -142,7 +144,7 @@ Open http://localhost:8000 in your browser. | `/api/input` | POST | Send input to terminal | | `/api/output` | POST | Poll for terminal output | | `/api/resize` | POST | Resize terminal dimensions | -| `/api/session` | DELETE | Close terminal session | +| `/api/session/close` | POST | Gracefully close terminal session | ## Project Structure @@ -197,7 +199,7 @@ This project is configured for deployment as a Databricks App. |----------|-------------| | `DATABRICKS_HOST` | Databricks workspace URL | | `DATABRICKS_TOKEN` | Your Personal Access Token (PAT) | -| `ANTHROPIC_MODEL` | Model name (default: `databricks-claude-sonnet-4-5`) | +| `ANTHROPIC_MODEL` | Model name (default: `databricks-claude-opus-4-6`) | ### Security Model diff --git a/app.py b/app.py index b61e4cd7..0648546a 100644 --- a/app.py +++ b/app.py @@ -77,19 +77,42 @@ def check_authorization(): def read_pty_output(session_id, fd): """Background thread to read PTY output into buffer.""" + with sessions_lock: + pid = sessions[session_id]["pid"] + while True: with sessions_lock: if session_id not in sessions: break try: - if select.select([fd], [], [], 0.1)[0]: - output = os.read(fd, 4096).decode(errors="replace") + readable, _, errors = select.select([fd], [], [fd], 0.5) + if readable or errors: + output = os.read(fd, 4096) + if not output: + # EOF — process exited + break with sessions_lock: if session_id in sessions: - sessions[session_id]["output_buffer"].append(output) + sessions[session_id]["output_buffer"].append(output.decode(errors="replace")) + else: + # select timed out — check if process is still alive + try: + pid_result, _ = os.waitpid(pid, os.WNOHANG) + if pid_result != 0: + # Process exited + break + except ChildProcessError: + # Process already reaped + break except OSError: break + # Process exited or fd closed — mark session as exited for the poll endpoint + with sessions_lock: + if session_id in sessions: + sessions[session_id]["exited"] = True + logger.info(f"Session {session_id} process exited") + def terminate_session(session_id, pid, master_fd): """Gracefully terminate a session: SIGHUP -> wait -> SIGKILL -> cleanup.""" @@ -247,12 +270,14 @@ def get_output(): if session_id not in sessions: return jsonify({"error": "Session not found"}), 404 - sessions[session_id]["last_poll_time"] = time.time() - buffer = sessions[session_id]["output_buffer"] + session = sessions[session_id] + session["last_poll_time"] = time.time() + buffer = session["output_buffer"] output = "".join(buffer) buffer.clear() + exited = session.get("exited", False) - return jsonify({"output": output}) + return jsonify({"output": output, "exited": exited}) @app.route("/api/resize", methods=["POST"]) @@ -277,20 +302,24 @@ def resize_terminal(): return jsonify({"error": str(e)}), 500 -@app.route("/api/session", methods=["DELETE"]) -def delete_session(): - """Close a terminal session.""" +@app.route("/api/session/close", methods=["POST"]) +def close_session(): + """Gracefully close a terminal session, killing the process.""" data = request.json session_id = data.get("session_id") - with sessions_lock: - if session_id in sessions: - try: - os.close(sessions[session_id]["master_fd"]) - except: - pass - del sessions[session_id] + if not session_id: + return jsonify({"error": "session_id required"}), 400 + with sessions_lock: + session = sessions.get(session_id) + if not session: + return jsonify({"status": "ok", "detail": "session not found"}) + pid = session["pid"] + master_fd = session["master_fd"] + + terminate_session(session_id, pid, master_fd) + logger.info(f"Session {session_id} closed by client") return jsonify({"status": "ok"}) diff --git a/app.yaml b/app.yaml index 345676fa..288b55e9 100644 --- a/app.yaml +++ b/app.yaml @@ -10,4 +10,4 @@ env: - name: DATABRICKS_TOKEN valueFrom: DATABRICKS_TOKEN - name: ANTHROPIC_MODEL - value: databricks-claude-sonnet-4-5 + value: databricks-claude-opus-4-6 \ No newline at end of file diff --git a/static/index.html b/static/index.html index 91246f9c..4bac5565 100644 --- a/static/index.html +++ b/static/index.html @@ -54,15 +54,36 @@ headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ session_id: sessionId }) }); + if (!resp.ok) { + // Session gone on server side — stop polling + cleanupSession(); + term.write('\r\n\x1b[31mSession ended.\x1b[0m\r\n'); + return; + } const data = await resp.json(); if (data.output) { term.write(data.output); } + if (data.exited) { + term.write('\r\n\x1b[33mShell process exited. You can close this tab.\x1b[0m\r\n'); + cleanupSession(); + } } catch (e) { console.error('Poll error:', e); } } + function cleanupSession() { + if (pollInterval) { + clearInterval(pollInterval); + pollInterval = null; + } + if (sessionId) { + navigator.sendBeacon('/api/session/close', JSON.stringify({ session_id: sessionId })); + sessionId = null; + } + } + async function init() { try { status.textContent = 'Initializing terminal...'; @@ -113,11 +134,7 @@ }); // Cleanup on page unload - window.addEventListener('beforeunload', () => { - if (sessionId) { - navigator.sendBeacon('/api/session', JSON.stringify({ session_id: sessionId })); - } - }); + window.addEventListener('beforeunload', () => cleanupSession()); } catch (e) { status.textContent = 'Error: ' + e.message; From f3e3d8c87e45f313e9c76a9f5a9cfe7f5f0c839f Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 08:24:19 -0500 Subject: [PATCH 004/382] feat: add Databricks AI Gateway support and refresh skills - setup_claude.py: use DATABRICKS_GATEWAY_HOST/anthropic when available - setup_gemini.py: use DATABRICKS_GATEWAY_HOST/gemini, add GEMINI_MODEL env var - setup_opencode.py: split into mlflow/v1 and openai/v1 providers on gateway, add GPT Codex models, add gemini-3-1-pro, remove llama - Replace all Databricks skills with latest from ai-dev-kit upstream - Add refresh-databricks-skills skill for on-demand updates Co-Authored-By: Claude Opus 4.6 --- .../agent-bricks/3-multi-agent-supervisors.md | 237 --- .claude/skills/agent-bricks/SKILL.md | 151 -- .../1-knowledge-assistants.md | 35 +- .../2-supervisor-agents.md | 394 +++++ .../skills/databricks-agent-bricks/SKILL.md | 211 +++ .../SKILL.md | 10 +- .claude/skills/databricks-app-apx/SKILL.md | 9 +- .../databricks-app-python/1-authorization.md | 150 ++ .../databricks-app-python/2-app-resources.md | 120 ++ .../databricks-app-python/3-frameworks.md | 246 +++ .../databricks-app-python/4-deployment.md | 142 ++ .../databricks-app-python/5-lakebase.md | 141 ++ .../databricks-app-python/6-mcp-approach.md | 94 ++ .../skills/databricks-app-python/README.md | 157 -- .claude/skills/databricks-app-python/SKILL.md | 888 ++--------- .claude/skills/databricks-app-python/dash.md | 553 ------- .../skills/databricks-app-python/streamlit.md | 790 ---------- .../SDP_guidance.md | 0 .../SKILL.md | 16 +- .../alerts_guidance.md | 0 .claude/skills/databricks-config/SKILL.md | 7 + .claude/skills/databricks-dbsql/SKILL.md | 300 ++++ .../skills/databricks-dbsql/ai-functions.md | 1348 +++++++++++++++++ .../skills/databricks-dbsql/best-practices.md | 475 ++++++ .../databricks-dbsql/geospatial-collations.md | 736 +++++++++ .../materialized-views-pipes.md | 676 +++++++++ .../skills/databricks-dbsql/sql-scripting.md | 1077 +++++++++++++ .claude/skills/databricks-docs/SKILL.md | 12 +- .claude/skills/databricks-genie/SKILL.md | 11 +- .claude/skills/databricks-genie/spaces.md | 4 +- .claude/skills/databricks-jobs/SKILL.md | 4 +- .claude/skills/databricks-jobs/task-types.md | 2 +- .../databricks-lakebase-autoscale/SKILL.md | 294 ++++ .../databricks-lakebase-autoscale/branches.md | 212 +++ .../databricks-lakebase-autoscale/computes.md | 208 +++ .../connection-patterns.md | 304 ++++ .../databricks-lakebase-autoscale/projects.md | 204 +++ .../reverse-etl.md | 177 +++ .../databricks-lakebase-provisioned/SKILL.md | 308 ++++ .../connection-patterns.md | 279 ++++ .../reverse-etl.md | 226 +++ .../skills/databricks-metric-views/SKILL.md | 229 +++ .../databricks-metric-views/patterns.md | 651 ++++++++ .../databricks-metric-views/yaml-reference.md | 338 +++++ .../databricks-mlflow-evaluation/SKILL.md | 148 ++ .../references/CRITICAL-interfaces.md | 61 + .../references/GOTCHAS.md | 267 ++++ .../patterns-context-optimization.md | 0 .../references/patterns-datasets.md | 0 .../references/patterns-evaluation.md | 0 .../references/patterns-judge-alignment.md | 316 ++++ .../patterns-prompt-optimization.md | 163 ++ .../references/patterns-scorers.md | 0 .../references/patterns-trace-analysis.md | 0 .../references/patterns-trace-ingestion.md | 680 +++++++++ .../references/user-journeys.md | 627 ++++++++ .../1-classical-ml.md | 0 .../2-custom-pyfunc.md | 0 .../3-genai-agents.md | 9 +- .../4-tools-integration.md | 1 - .../5-development-testing.md | 0 .../6-logging-registration.md | 0 .../7-deployment.md | 76 +- .../8-querying-endpoints.md | 0 .../9-package-requirements.md | 0 .../SKILL.md | 69 +- .claude/skills/databricks-python-sdk/SKILL.md | 10 + .../1-ingestion-patterns.md | 130 +- .../10-mcp-approach.md | 173 +++ .../2-streaming-patterns.md | 9 +- .../3-scd-query-patterns.md} | 78 +- .../4-performance-tuning.md | 0 .../5-python-api.md | 0 .../6-dlt-migration.md | 8 +- .../7-advanced-configuration.md | 0 .../8-project-initialization.md | 107 +- .../9-auto_cdc.md | 353 +++++ .../SKILL.md | 577 +++++++ .../SKILL.md | 65 + .../checkpoint-best-practices.md | 316 ++++ .../kafka-streaming.md | 417 +++++ .../merge-operations.md | 358 +++++ .../multi-sink-writes.md | 427 ++++++ .../stateful-operations.md | 397 +++++ .../stream-static-joins.md | 519 +++++++ .../stream-stream-joins.md | 588 +++++++ .../streaming-best-practices.md | 265 ++++ .../trigger-and-cost-optimization.md | 517 +++++++ .../SKILL.md | 8 +- .../skills/databricks-unity-catalog/SKILL.md | 7 + .../SKILL.md | 9 +- .../skills/databricks-vector-search/SKILL.md | 357 +++++ .../databricks-vector-search/index-types.md | 254 ++++ .../1-setup-and-authentication.md | 199 +++ .../2-python-client.md | 323 ++++ .../3-multilanguage-clients.md | 314 ++++ .../4-protobuf-schema.md | 191 +++ .../5-operations-and-limits.md | 251 +++ .../skills/databricks-zerobus-ingest/SKILL.md | 228 +++ .claude/skills/mlflow-evaluation/SKILL.md | 95 -- .../references/user-journeys.md | 332 ---- .../skills/refresh-databricks-skills/SKILL.md | 59 + .../spark-declarative-pipelines/SKILL.md | 474 ------ .../skills/spark-python-data-source/SKILL.md | 311 ++++ .../references/authentication-patterns.md | 361 +++++ .../references/error-handling.md | 432 ++++++ .../references/partitioning-patterns.md | 319 ++++ .../references/production-patterns.md | 475 ++++++ .../references/streaming-patterns.md | 400 +++++ .../references/testing-patterns.md | 439 ++++++ .../references/type-conversion.md | 370 +++++ app.yaml | 9 +- setup_claude.py | 19 +- setup_gemini.py | 99 ++ setup_opencode.py | 221 +++ 115 files changed, 22984 insertions(+), 3729 deletions(-) delete mode 100644 .claude/skills/agent-bricks/3-multi-agent-supervisors.md delete mode 100644 .claude/skills/agent-bricks/SKILL.md rename .claude/skills/{agent-bricks => databricks-agent-bricks}/1-knowledge-assistants.md (80%) create mode 100644 .claude/skills/databricks-agent-bricks/2-supervisor-agents.md create mode 100644 .claude/skills/databricks-agent-bricks/SKILL.md rename .claude/skills/{aibi-dashboards => databricks-aibi-dashboards}/SKILL.md (98%) create mode 100644 .claude/skills/databricks-app-python/1-authorization.md create mode 100644 .claude/skills/databricks-app-python/2-app-resources.md create mode 100644 .claude/skills/databricks-app-python/3-frameworks.md create mode 100644 .claude/skills/databricks-app-python/4-deployment.md create mode 100644 .claude/skills/databricks-app-python/5-lakebase.md create mode 100644 .claude/skills/databricks-app-python/6-mcp-approach.md delete mode 100644 .claude/skills/databricks-app-python/README.md delete mode 100644 .claude/skills/databricks-app-python/dash.md delete mode 100644 .claude/skills/databricks-app-python/streamlit.md rename .claude/skills/{asset-bundles => databricks-asset-bundles}/SDP_guidance.md (100%) rename .claude/skills/{asset-bundles => databricks-asset-bundles}/SKILL.md (89%) rename .claude/skills/{asset-bundles => databricks-asset-bundles}/alerts_guidance.md (100%) create mode 100644 .claude/skills/databricks-dbsql/SKILL.md create mode 100644 .claude/skills/databricks-dbsql/ai-functions.md create mode 100644 .claude/skills/databricks-dbsql/best-practices.md create mode 100644 .claude/skills/databricks-dbsql/geospatial-collations.md create mode 100644 .claude/skills/databricks-dbsql/materialized-views-pipes.md create mode 100644 .claude/skills/databricks-dbsql/sql-scripting.md create mode 100644 .claude/skills/databricks-lakebase-autoscale/SKILL.md create mode 100644 .claude/skills/databricks-lakebase-autoscale/branches.md create mode 100644 .claude/skills/databricks-lakebase-autoscale/computes.md create mode 100644 .claude/skills/databricks-lakebase-autoscale/connection-patterns.md create mode 100644 .claude/skills/databricks-lakebase-autoscale/projects.md create mode 100644 .claude/skills/databricks-lakebase-autoscale/reverse-etl.md create mode 100644 .claude/skills/databricks-lakebase-provisioned/SKILL.md create mode 100644 .claude/skills/databricks-lakebase-provisioned/connection-patterns.md create mode 100644 .claude/skills/databricks-lakebase-provisioned/reverse-etl.md create mode 100644 .claude/skills/databricks-metric-views/SKILL.md create mode 100644 .claude/skills/databricks-metric-views/patterns.md create mode 100644 .claude/skills/databricks-metric-views/yaml-reference.md create mode 100644 .claude/skills/databricks-mlflow-evaluation/SKILL.md rename .claude/skills/{mlflow-evaluation => databricks-mlflow-evaluation}/references/CRITICAL-interfaces.md (87%) rename .claude/skills/{mlflow-evaluation => databricks-mlflow-evaluation}/references/GOTCHAS.md (54%) rename .claude/skills/{mlflow-evaluation => databricks-mlflow-evaluation}/references/patterns-context-optimization.md (100%) rename .claude/skills/{mlflow-evaluation => databricks-mlflow-evaluation}/references/patterns-datasets.md (100%) rename .claude/skills/{mlflow-evaluation => databricks-mlflow-evaluation}/references/patterns-evaluation.md (100%) create mode 100644 .claude/skills/databricks-mlflow-evaluation/references/patterns-judge-alignment.md create mode 100644 .claude/skills/databricks-mlflow-evaluation/references/patterns-prompt-optimization.md rename .claude/skills/{mlflow-evaluation => databricks-mlflow-evaluation}/references/patterns-scorers.md (100%) rename .claude/skills/{mlflow-evaluation => databricks-mlflow-evaluation}/references/patterns-trace-analysis.md (100%) create mode 100644 .claude/skills/databricks-mlflow-evaluation/references/patterns-trace-ingestion.md create mode 100644 .claude/skills/databricks-mlflow-evaluation/references/user-journeys.md rename .claude/skills/{model-serving => databricks-model-serving}/1-classical-ml.md (100%) rename .claude/skills/{model-serving => databricks-model-serving}/2-custom-pyfunc.md (100%) rename .claude/skills/{model-serving => databricks-model-serving}/3-genai-agents.md (95%) rename .claude/skills/{model-serving => databricks-model-serving}/4-tools-integration.md (98%) rename .claude/skills/{model-serving => databricks-model-serving}/5-development-testing.md (100%) rename .claude/skills/{model-serving => databricks-model-serving}/6-logging-registration.md (100%) rename .claude/skills/{model-serving => databricks-model-serving}/7-deployment.md (66%) rename .claude/skills/{model-serving => databricks-model-serving}/8-querying-endpoints.md (100%) rename .claude/skills/{model-serving => databricks-model-serving}/9-package-requirements.md (100%) rename .claude/skills/{model-serving => databricks-model-serving}/SKILL.md (62%) rename .claude/skills/{spark-declarative-pipelines => databricks-spark-declarative-pipelines}/1-ingestion-patterns.md (69%) create mode 100644 .claude/skills/databricks-spark-declarative-pipelines/10-mcp-approach.md rename .claude/skills/{spark-declarative-pipelines => databricks-spark-declarative-pipelines}/2-streaming-patterns.md (95%) rename .claude/skills/{spark-declarative-pipelines/3-scd-patterns.md => databricks-spark-declarative-pipelines/3-scd-query-patterns.md} (67%) rename .claude/skills/{spark-declarative-pipelines => databricks-spark-declarative-pipelines}/4-performance-tuning.md (100%) rename .claude/skills/{spark-declarative-pipelines => databricks-spark-declarative-pipelines}/5-python-api.md (100%) rename .claude/skills/{spark-declarative-pipelines => databricks-spark-declarative-pipelines}/6-dlt-migration.md (96%) rename .claude/skills/{spark-declarative-pipelines => databricks-spark-declarative-pipelines}/7-advanced-configuration.md (100%) rename .claude/skills/{spark-declarative-pipelines => databricks-spark-declarative-pipelines}/8-project-initialization.md (87%) create mode 100644 .claude/skills/databricks-spark-declarative-pipelines/9-auto_cdc.md create mode 100644 .claude/skills/databricks-spark-declarative-pipelines/SKILL.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/SKILL.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/checkpoint-best-practices.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/kafka-streaming.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/merge-operations.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/multi-sink-writes.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/stateful-operations.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/stream-static-joins.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/stream-stream-joins.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/streaming-best-practices.md create mode 100644 .claude/skills/databricks-spark-structured-streaming/trigger-and-cost-optimization.md rename .claude/skills/{synthetic-data-generation => databricks-synthetic-data-generation}/SKILL.md (98%) rename .claude/skills/{unstructured-pdf-generation => databricks-unstructured-pdf-generation}/SKILL.md (91%) create mode 100644 .claude/skills/databricks-vector-search/SKILL.md create mode 100644 .claude/skills/databricks-vector-search/index-types.md create mode 100644 .claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md create mode 100644 .claude/skills/databricks-zerobus-ingest/2-python-client.md create mode 100644 .claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md create mode 100644 .claude/skills/databricks-zerobus-ingest/4-protobuf-schema.md create mode 100644 .claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md create mode 100644 .claude/skills/databricks-zerobus-ingest/SKILL.md delete mode 100644 .claude/skills/mlflow-evaluation/SKILL.md delete mode 100644 .claude/skills/mlflow-evaluation/references/user-journeys.md create mode 100644 .claude/skills/refresh-databricks-skills/SKILL.md delete mode 100644 .claude/skills/spark-declarative-pipelines/SKILL.md create mode 100644 .claude/skills/spark-python-data-source/SKILL.md create mode 100644 .claude/skills/spark-python-data-source/references/authentication-patterns.md create mode 100644 .claude/skills/spark-python-data-source/references/error-handling.md create mode 100644 .claude/skills/spark-python-data-source/references/partitioning-patterns.md create mode 100644 .claude/skills/spark-python-data-source/references/production-patterns.md create mode 100644 .claude/skills/spark-python-data-source/references/streaming-patterns.md create mode 100644 .claude/skills/spark-python-data-source/references/testing-patterns.md create mode 100644 .claude/skills/spark-python-data-source/references/type-conversion.md create mode 100644 setup_gemini.py create mode 100644 setup_opencode.py diff --git a/.claude/skills/agent-bricks/3-multi-agent-supervisors.md b/.claude/skills/agent-bricks/3-multi-agent-supervisors.md deleted file mode 100644 index c546d242..00000000 --- a/.claude/skills/agent-bricks/3-multi-agent-supervisors.md +++ /dev/null @@ -1,237 +0,0 @@ -# Multi-Agent Supervisors (MAS) - -Multi-Agent Supervisors orchestrate multiple specialized agents, routing user queries to the most appropriate agent based on the query content. - -## What is a Multi-Agent Supervisor? - -A MAS acts as a traffic controller for multiple AI agents. When a user asks a question: - -1. **Analyzes** the query to understand the intent -2. **Routes** to the most appropriate specialized agent -3. **Returns** the agent's response to the user - -This allows you to combine multiple specialized agents into a single unified interface. - -## When to Use - -Use a Multi-Agent Supervisor when: -- You have multiple specialized agents (billing, technical support, HR, etc.) -- Users shouldn't need to know which agent to ask -- You want to provide a unified conversational experience - -## Prerequisites - -Before creating a MAS, you need agents of one or both types: - -**Model Serving Endpoints** (`endpoint_name`): -- Knowledge Assistant (KA) endpoints (e.g., `ka-abc123-endpoint`) -- Custom agents built with LangChain, LlamaIndex, etc. -- Fine-tuned models -- RAG applications - -**Genie Spaces** (`genie_space_id`): -- Existing Genie spaces for SQL-based data exploration -- Great for analytics, metrics, and data-driven questions -- No separate endpoint deployment required - reference the space directly -- To find a Genie space by name, use `find_genie_by_name(display_name="My Genie")` -- **Note**: There is NO system table for Genie spaces - do not try to query `system.ai.genie_spaces` - -## Creating a Multi-Agent Supervisor - -Use the `create_or_update_mas` tool: - -- `name`: "Customer Support MAS" -- `agents`: - ```json - [ - { - "name": "policy_agent", - "ka_tile_id": "f32c5f73-466b-4798-b3a0-5396b5ece2a5", - "description": "Answers questions about company policies and procedures from indexed documents" - }, - { - "name": "usage_analytics", - "genie_space_id": "01abc123-def4-5678-90ab-cdef12345678", - "description": "Answers data questions about usage metrics, trends, and statistics" - }, - { - "name": "custom_agent", - "endpoint_name": "my-custom-endpoint", - "description": "Handles specialized queries via custom model endpoint" - } - ] - ``` -- `description`: "Routes customer queries to specialized support agents" -- `instructions`: "Analyze the user's question and route to the most appropriate agent. If unclear, ask for clarification." - -This example shows mixing Knowledge Assistants (policy_agent), Genie spaces (usage_analytics), and custom endpoints (custom_agent). - -## Agent Configuration - -Each agent in the `agents` list needs: - -| Field | Required | Description | -|-------|----------|-------------| -| `name` | Yes | Internal identifier for the agent | -| `description` | Yes | What this agent handles (critical for routing) | -| `ka_tile_id` | One of these | Knowledge Assistant tile ID (for document Q&A agents) | -| `genie_space_id` | One of these | Genie space ID (for SQL-based data agents) | -| `endpoint_name` | One of these | Model serving endpoint name (for custom agents) | - -**Note**: Provide exactly one of: `ka_tile_id`, `genie_space_id`, or `endpoint_name`. - -To find a KA tile_id, use `find_ka_by_name(name="Your KA Name")`. -To find a Genie space_id, use `find_genie_by_name(display_name="Your Genie Name")`. - -### Writing Good Descriptions - -The `description` field is critical for routing. Make it specific: - -**Good descriptions:** -- "Handles billing questions including invoices, payments, refunds, and subscription changes" -- "Answers technical questions about API errors, integration issues, and product bugs" -- "Provides information about HR policies, PTO, benefits, and employee handbook" - -**Bad descriptions:** -- "Billing agent" (too vague) -- "Handles stuff" (not helpful) -- "Technical" (not specific) - -## Provisioning Timeline - -After creation, the MAS endpoint needs to provision: - -| Status | Meaning | Duration | -|--------|---------|----------| -| `PROVISIONING` | Creating the supervisor | 2-5 minutes | -| `ONLINE` | Ready to route queries | - | -| `OFFLINE` | Not currently running | - | - -Use `get_mas` to check the status. - -## Adding Example Questions - -Example questions help with evaluation and can guide routing optimization: - -```json -{ - "examples": [ - { - "question": "I haven't received my invoice for this month", - "guideline": "Should be routed to billing_agent" - }, - { - "question": "The API is returning a 500 error", - "guideline": "Should be routed to technical_agent" - }, - { - "question": "How many vacation days do I have?", - "guideline": "Should be routed to hr_agent" - } - ] -} -``` - -If the MAS is not yet `ONLINE`, examples are queued and added automatically when ready. - -## Best Practices - -### Agent Design - -1. **Specialized agents**: Each agent should have a clear, distinct purpose -2. **Non-overlapping domains**: Avoid agents with similar descriptions -3. **Clear boundaries**: Define what each agent does and doesn't handle - -### Instructions - -Provide routing instructions: - -``` -You are a customer support supervisor. Your job is to route user queries to the right specialist: - -1. For billing, payments, or subscription questions → billing_agent -2. For technical issues, bugs, or API problems → technical_agent -3. For HR, benefits, or policy questions → hr_agent - -If the query is unclear or spans multiple domains, ask the user to clarify. -``` - -### Fallback Handling - -Consider adding a general-purpose agent for queries that don't fit elsewhere: - -```json -{ - "name": "general_agent", - "endpoint_name": "general-support-endpoint", - "description": "Handles general inquiries that don't fit other categories, provides navigation help" -} -``` - -## Example Workflow - -1. **Deploy specialized agents** as model serving endpoints: - - `billing-assistant-endpoint` - - `tech-support-endpoint` - - `hr-assistant-endpoint` - -2. **Create the MAS**: - - Configure agents with clear descriptions - - Add routing instructions - -3. **Wait for ONLINE status** (2-5 minutes) - -4. **Add example questions** for evaluation - -5. **Test routing** with various query types - -## Updating a Multi-Agent Supervisor - -To update an existing MAS: - -1. **Add/remove agents**: Call `create_or_update_mas` with updated `agents` list -2. **Update descriptions**: Change agent descriptions to improve routing -3. **Modify instructions**: Update routing rules - -The tool finds the existing MAS by name and updates it. - -## Troubleshooting - -### Queries routed to wrong agent - -- Review and improve agent descriptions -- Make descriptions more specific and distinct -- Add examples that demonstrate correct routing - -### Endpoint not responding - -- Verify each underlying model serving endpoint is running -- Check endpoint logs for errors -- Ensure endpoints accept the expected input format - -### Slow responses - -- Check latency of underlying endpoints -- Consider endpoint scaling settings -- Monitor for cold start issues - -## Advanced: Hierarchical Routing - -For complex scenarios, you can create multiple levels of MAS: - -``` -Top-level MAS -├── Customer Support MAS -│ ├── billing_agent -│ ├── technical_agent -│ └── general_agent -├── Sales MAS -│ ├── pricing_agent -│ ├── demo_agent -│ └── contract_agent -└── Internal MAS - ├── hr_agent - └── it_helpdesk_agent -``` - -Each sub-MAS is deployed as an endpoint and configured as an agent in the top-level MAS. diff --git a/.claude/skills/agent-bricks/SKILL.md b/.claude/skills/agent-bricks/SKILL.md deleted file mode 100644 index bd8f8488..00000000 --- a/.claude/skills/agent-bricks/SKILL.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -name: agent-bricks -description: "Create and manage Databricks Agent Bricks: Knowledge Assistants (KA) for document Q&A, Genie Spaces for SQL exploration, and Multi-Agent Supervisors (MAS) for multi-agent orchestration. Use when building conversational AI applications on Databricks." ---- - -# Agent Bricks - -Create and manage Databricks Agent Bricks - pre-built AI components for building conversational applications. - -## Overview - -Agent Bricks are three types of pre-built AI tiles in Databricks: - -| Brick | Purpose | Data Source | -|-------|---------|-------------| -| **Knowledge Assistant (KA)** | Document-based Q&A using RAG | PDF/text files in Volumes | -| **Genie Space** | Natural language to SQL | Unity Catalog tables | -| **Multi-Agent Supervisor (MAS)** | Multi-agent orchestration | Model serving endpoints | - -## Prerequisites - -Before creating Agent Bricks, ensure you have the required data: - -### For Knowledge Assistants -- **Documents in a Volume**: PDF, text, or other files stored in a Unity Catalog volume -- Generate synthetic documents using the `unstructured-pdf-generation` skill if needed - -### For Genie Spaces -- **See the `databricks-genie` skill** for comprehensive Genie Space guidance -- Tables in Unity Catalog with the data to explore -- Generate raw data using the `synthetic-data-generation` skill -- Create tables using the `spark-declarative-pipelines` skill - -### For Multi-Agent Supervisors -- **Model Serving Endpoints**: Deployed agent endpoints (KA endpoints, custom agents, fine-tuned models) -- **Genie Spaces**: Existing Genie spaces can be used directly as agents for SQL-based queries -- Mix and match endpoint-based and Genie-based agents in the same MAS - -## MCP Tools - -### Knowledge Assistant Tools - -**create_or_update_ka** - Create or update a Knowledge Assistant -- `name`: Name for the KA -- `volume_path`: Path to documents (e.g., `/Volumes/catalog/schema/volume/folder`) -- `description`: (optional) What the KA does -- `instructions`: (optional) How the KA should answer -- `tile_id`: (optional) Existing tile_id to update -- `add_examples_from_volume`: (optional, default: true) Auto-add examples from JSON files - -**get_ka** - Get Knowledge Assistant details -- `tile_id`: The KA tile ID - -**find_ka_by_name** - Find a Knowledge Assistant by name -- `name`: The exact name of the KA to find -- Returns: `tile_id`, `name`, `endpoint_name`, `endpoint_status` -- Use this to look up an existing KA when you know the name but not the tile_id - -**delete_ka** - Delete a Knowledge Assistant -- `tile_id`: The KA tile ID to delete - -### Genie Space Tools - -**For comprehensive Genie guidance, use the `databricks-genie` skill.** - -Basic tools available: - -- `create_or_update_genie` - Create or update a Genie Space -- `get_genie` - Get Genie Space details -- `delete_genie` - Delete a Genie Space - -See `databricks-genie` skill for: -- Table inspection workflow -- Sample question best practices -- Curation (instructions, certified queries) - -**IMPORTANT**: There is NO system table for Genie spaces (e.g., `system.ai.genie_spaces` does not exist). To find a Genie space by name, use the `find_genie_by_name` tool. - -### Multi-Agent Supervisor Tools - -**create_or_update_mas** - Create or update a Multi-Agent Supervisor -- `name`: Name for the MAS -- `agents`: List of agent configurations, each with: - - `name`: Agent identifier (required) - - `description`: What this agent handles - critical for routing (required) - - `ka_tile_id`: Knowledge Assistant tile ID (use for document Q&A agents - recommended for KAs) - - `genie_space_id`: Genie space ID (use for SQL-based data agents) - - `endpoint_name`: Model serving endpoint name (use for custom agents) - - Note: Provide exactly one of: `ka_tile_id`, `genie_space_id`, or `endpoint_name` -- `description`: (optional) What the MAS does -- `instructions`: (optional) Routing instructions for the supervisor -- `tile_id`: (optional) Existing tile_id to update -- `examples`: (optional) List of example questions with `question` and `guideline` fields - -**get_mas** - Get Multi-Agent Supervisor details -- `tile_id`: The MAS tile ID - -**find_mas_by_name** - Find a Multi-Agent Supervisor by name -- `name`: The exact name of the MAS to find -- Returns: `tile_id`, `name`, `endpoint_status`, `agents_count` -- Use this to look up an existing MAS when you know the name but not the tile_id - -**delete_mas** - Delete a Multi-Agent Supervisor -- `tile_id`: The MAS tile ID to delete - -## Typical Workflow - -### 1. Generate Source Data - -Before creating Agent Bricks, generate the required source data: - -**For KA (document Q&A)**: -``` -1. Use `unstructured-pdf-generation` skill to generate PDFs -2. PDFs are saved to a Volume with companion JSON files (question/guideline pairs) -``` - -**For Genie (SQL exploration)**: -``` -1. Use `synthetic-data-generation` skill to create raw parquet data -2. Use `spark-declarative-pipelines` skill to create bronze/silver/gold tables -``` - -### 2. Create the Agent Brick - -Use the appropriate `create_or_update_*` tool with your data sources. - -### 3. Wait for Provisioning - -Newly created KA and MAS tiles need time to provision. The endpoint status will progress: -- `PROVISIONING` - Being created (can take 2-5 minutes) -- `ONLINE` - Ready to use -- `OFFLINE` - Not running - -### 4. Add Examples (Automatic) - -For KA, if `add_examples_from_volume=true`, examples are automatically extracted from JSON files in the volume and added once the endpoint is `ONLINE`. - -## Best Practices - -1. **Use meaningful names**: Names are sanitized automatically (spaces become underscores) -2. **Provide descriptions**: Helps users understand what the brick does -3. **Add instructions**: Guide the AI's behavior and tone -4. **Include sample questions**: Shows users how to interact with the brick -5. **Use the workflow**: Generate data first, then create the brick - -## See Also - -- `1-knowledge-assistants.md` - Detailed KA patterns and examples -- `databricks-genie` skill - Detailed Genie patterns, curation, and examples -- `3-multi-agent-supervisors.md` - Detailed MAS patterns and examples diff --git a/.claude/skills/agent-bricks/1-knowledge-assistants.md b/.claude/skills/databricks-agent-bricks/1-knowledge-assistants.md similarity index 80% rename from .claude/skills/agent-bricks/1-knowledge-assistants.md rename to .claude/skills/databricks-agent-bricks/1-knowledge-assistants.md index d81e351d..3adff469 100644 --- a/.claude/skills/agent-bricks/1-knowledge-assistants.md +++ b/.claude/skills/databricks-agent-bricks/1-knowledge-assistants.md @@ -25,12 +25,12 @@ Before creating a KA, you need documents in a Unity Catalog Volume: - Upload PDFs/text files to a Volume manually or via SDK **Option 2: Generate synthetic documents** -- Use the `unstructured-pdf-generation` skill to create realistic PDF documents +- Use the `databricks-unstructured-pdf-generation` skill to create realistic PDF documents - Each PDF gets a companion JSON file with question/guideline pairs for evaluation ## Creating a Knowledge Assistant -Use the `create_or_update_ka` tool: +Use the `manage_ka` tool with `action="create_or_update"`: - `name`: "HR Policy Assistant" - `volume_path`: "/Volumes/my_catalog/my_schema/raw_data/hr_docs" @@ -52,7 +52,7 @@ After creation, the KA endpoint needs to provision: | `ONLINE` | Ready to use | - | | `OFFLINE` | Not currently running | - | -Use `get_ka` to check the status: +Use `manage_ka` with `action="get"` to check the status: - `tile_id`: "" @@ -76,7 +76,7 @@ These are automatically added when `add_examples_from_volume=true` (default). ### Manual -Examples can also be specified in the `create_or_update_ka` call if needed. +Examples can also be specified in the `manage_ka` create_or_update call if needed. ## Best Practices @@ -101,12 +101,12 @@ Be helpful and professional. When answering: To update the indexed documents: 1. Add/remove/modify files in the volume -2. Call `create_or_update_ka` with the same name and `tile_id` +2. Call `manage_ka` with `action="create_or_update"`, the same name and `tile_id` 3. The KA will re-index the updated content ## Example Workflow -1. **Generate PDF documents** using `unstructured-pdf-generation` skill: +1. **Generate PDF documents** using `databricks-unstructured-pdf-generation` skill: - Creates PDFs in `/Volumes/catalog/schema/raw_data/pdf_documents` - Creates JSON files with question/guideline pairs @@ -120,13 +120,13 @@ To update the indexed documents: 5. **Test the KA** in the Databricks UI -## Using KA in Multi-Agent Supervisors +## Using KA in Supervisor Agents -Knowledge Assistants can be used as agents in a Multi-Agent Supervisor (MAS). Each KA has an associated model serving endpoint. +Knowledge Assistants can be used as agents in a Supervisor Agent (formerly Multi-Agent Supervisor, MAS). Each KA has an associated model serving endpoint. ### Finding the Endpoint Name -Use `get_ka` to retrieve the KA details. The response includes: +Use `manage_ka` with `action="get"` to retrieve the KA details. The response includes: - `tile_id`: The unique identifier for the KA - `name`: The KA name (sanitized) - `endpoint_status`: Current status (ONLINE, PROVISIONING, etc.) @@ -135,26 +135,27 @@ The endpoint name follows this pattern: `ka-{tile_id}-endpoint` ### Finding a KA by Name -If you know the KA name but not the tile_id, use `find_ka_by_name`: +If you know the KA name but not the tile_id, use `manage_ka` with `action="find_by_name"`: ```python -find_ka_by_name(name="HR_Policy_Assistant") +manage_ka(action="find_by_name", name="HR_Policy_Assistant") # Returns: {"found": True, "tile_id": "01abc...", "name": "HR_Policy_Assistant", "endpoint_name": "ka-01abc...-endpoint"} ``` -### Example: Adding KA to MAS +### Example: Adding KA to Supervisor Agent ```python # First, find the KA -ka_result = find_ka_by_name(name="HR_Policy_Assistant") +manage_ka(action="find_by_name", name="HR_Policy_Assistant") -# Then use it in a MAS -create_or_update_mas( - name="Support MAS", +# Then use the tile_id in a Supervisor Agent +manage_mas( + action="create_or_update", + name="Support_MAS", agents=[ { "name": "hr_agent", - "endpoint_name": ka_result["endpoint_name"], + "ka_tile_id": "", "description": "Answers HR policy questions from the employee handbook" } ] diff --git a/.claude/skills/databricks-agent-bricks/2-supervisor-agents.md b/.claude/skills/databricks-agent-bricks/2-supervisor-agents.md new file mode 100644 index 00000000..7121bfcf --- /dev/null +++ b/.claude/skills/databricks-agent-bricks/2-supervisor-agents.md @@ -0,0 +1,394 @@ +# Supervisor Agents (MAS) + +Supervisor Agents orchestrate multiple specialized agents, routing user queries to the most appropriate agent based on the query content. + +## What is a Supervisor Agent? + +A Supervisor Agent (formerly Multi-Agent Supervisor, MAS) acts as a traffic controller for multiple AI agents, routing user queries to the most appropriate agent. It supports five types of agents: + +1. **Knowledge Assistants (KA)**: Document-based Q&A from PDFs/files in Volumes +2. **Genie Spaces**: Natural language to SQL for data exploration +3. **Model Serving Endpoints**: Custom LLM agents, fine-tuned models, RAG applications +4. **Unity Catalog Functions**: Callable UC functions for data operations +5. **External MCP Servers**: JSON-RPC endpoints via UC HTTP Connections for external system integration + +When a user asks a question: +1. **Analyzes** the query to understand the intent +2. **Routes** to the most appropriate specialized agent +3. **Returns** the agent's response to the user + +This allows you to combine multiple specialized agents into a single unified interface. + +## When to Use + +Use a Supervisor Agent when: +- You have multiple specialized agents (billing, technical support, HR, etc.) +- Users shouldn't need to know which agent to ask +- You want to provide a unified conversational experience + +## Prerequisites + +Before creating a Supervisor Agent, you need agents of one or both types: + +**Model Serving Endpoints** (`endpoint_name`): +- Knowledge Assistant (KA) endpoints (e.g., `ka-abc123-endpoint`) +- Custom agents built with LangChain, LlamaIndex, etc. +- Fine-tuned models +- RAG applications + +**Genie Spaces** (`genie_space_id`): +- Existing Genie spaces for SQL-based data exploration +- Great for analytics, metrics, and data-driven questions +- No separate endpoint deployment required - reference the space directly +- To find a Genie space by name, use `find_genie_by_name(display_name="My Genie")` +- **Note**: There is NO system table for Genie spaces - do not try to query `system.ai.genie_spaces` + +## Unity Catalog Functions + +Unity Catalog Functions allow Supervisor Agents to call registered UC functions for data operations. + +### Prerequisites + +- UC Function already exists (use SQL `CREATE FUNCTION` or Python UDF) +- Agent service principal has `EXECUTE` privilege: + ```sql + GRANT EXECUTE ON FUNCTION catalog.schema.function_name TO ``; + ``` + +### Configuration + +```json +{ + "name": "data_enrichment", + "uc_function_name": "sales_analytics.utils.enrich_customer_data", + "description": "Enriches customer records with demographic and purchase history data" +} +``` + +**Field**: `uc_function_name` - Fully-qualified function name in format `catalog.schema.function_name` + +## External MCP Servers + +External MCP Servers enable Supervisor Agents to interact with external systems (ERP, CRM, etc.) via UC HTTP Connections. The MCP server implements a JSON-RPC 2.0 endpoint that exposes tools for the Supervisor Agent to call. + +### Prerequisites + +**1. MCP Server Endpoint**: Your external system must provide a JSON-RPC 2.0 endpoint (e.g., `/api/mcp`) that implements the MCP protocol: + +```python +# Example MCP server tool definition +TOOLS = [ + { + "name": "approve_invoice", + "description": "Approve a specific invoice", + "inputSchema": { + "type": "object", + "properties": { + "invoice_number": {"type": "string", "description": "Invoice number to approve"}, + "approver": {"type": "string", "description": "Name/email of approver"}, + }, + "required": ["invoice_number"], + }, + }, +] + +# JSON-RPC methods: initialize, tools/list, tools/call +``` + +**2. UC HTTP Connection**: Create a Unity Catalog HTTP Connection that points to your MCP endpoint: + +```sql +CREATE CONNECTION my_mcp_connection TYPE HTTP +OPTIONS ( + host 'https://my-app.databricksapps.com', -- Your MCP server URL + port '443', + base_path '/api/mcp', -- Path to JSON-RPC endpoint + client_id '', -- OAuth M2M credentials + client_secret '', + oauth_scope 'all-apis', + token_endpoint 'https://.azuredatabricks.net/oidc/v1/token', + is_mcp_connection 'true' -- REQUIRED: Identifies as MCP connection +); +``` + +**3. Grant Permissions**: Agent service principal needs access to the connection: + +```sql +GRANT USE CONNECTION ON my_mcp_connection TO ``; +``` + +### Configuration + +Reference the UC Connection using the `connection_name` field: + +```python +{ + "name": "external_operations", + "connection_name": "my_mcp_connection", + "description": "Execute external system operations: approve invoices, create records, trigger workflows" +} +``` + +**Field**: `connection_name` - the name of the Unity Catalog HTTP Connection configured as an MCP server + +**Important**: Make the description comprehensive - it guides the Supervisor Agent's routing decisions for when to call this agent. + +### Complete Example: Multi-System Supervisor + +Example showing integration of Genie, KA, and external MCP: + +```python +manage_mas( + action="create_or_update", + name="AP_Invoice_Supervisor", + agents=[ + { + "name": "billing_analyst", + "genie_space_id": "01abc123...", + "description": "SQL analytics on AP invoice data: spending trends, vendor analysis, aging reports" + }, + { + "name": "policy_expert", + "ka_tile_id": "f32c5f73...", + "description": "Answers questions about AP policies, approval workflows, and compliance requirements from policy documents" + }, + { + "name": "ap_operations", + "connection_name": "ap_invoice_mcp", + "description": ( + "Execute AP operations: approve/reject/flag invoices, search invoice details, " + "get vendor summaries, trigger batch workflows. Use for ANY action or write operation." + ) + } + ], + description="AP automation assistant with analytics, policy guidance, and operational actions", + instructions=""" + Route queries as follows: + - Data questions (invoice counts, spend analysis, vendor metrics) → billing_analyst + - Policy questions (thresholds, SLAs, compliance rules) → policy_expert + - Actions (approve, reject, flag, search, workflows) → ap_operations + + When a user asks to approve, reject, or flag an invoice, ALWAYS use ap_operations. + """ +) +``` + +### MCP Connection Testing + +Verify your connection before adding to MAS: + +```sql +-- Test tools/list method +SELECT http_request( + conn => 'my_mcp_connection', + method => 'POST', + path => '', + json => '{"jsonrpc":"2.0","method":"tools/list","id":1}' +); +``` + +### Resources + +- **MCP Protocol Spec**: [Model Context Protocol](https://modelcontextprotocol.io) + +## Creating a Supervisor Agent + +Use the `manage_mas` tool with `action="create_or_update"`: + +- `name`: "Customer Support MAS" +- `agents`: + ```json + [ + { + "name": "policy_agent", + "ka_tile_id": "f32c5f73-466b-4798-b3a0-5396b5ece2a5", + "description": "Answers questions about company policies and procedures from indexed documents" + }, + { + "name": "usage_analytics", + "genie_space_id": "01abc123-def4-5678-90ab-cdef12345678", + "description": "Answers data questions about usage metrics, trends, and statistics" + }, + { + "name": "custom_agent", + "endpoint_name": "my-custom-endpoint", + "description": "Handles specialized queries via custom model endpoint" + } + ] + ``` +- `description`: "Routes customer queries to specialized support agents" +- `instructions`: "Analyze the user's question and route to the most appropriate agent. If unclear, ask for clarification." + +This example shows mixing Knowledge Assistants (policy_agent), Genie spaces (usage_analytics), and custom endpoints (custom_agent). + +## Agent Configuration + +Each agent in the `agents` list needs: + +| Field | Required | Description | +|-------|----------|-------------| +| `name` | Yes | Internal identifier for the agent | +| `description` | Yes | What this agent handles (critical for routing) | +| `ka_tile_id` | One of these | Knowledge Assistant tile ID (for document Q&A agents) | +| `genie_space_id` | One of these | Genie space ID (for SQL-based data agents) | +| `endpoint_name` | One of these | Model serving endpoint name (for custom agents) | +| `uc_function_name` | One of these | Unity Catalog function name in format `catalog.schema.function_name` | +| `connection_name` | One of these | Unity Catalog connection name (for external MCP servers) | + +**Note**: Provide exactly one of: `ka_tile_id`, `genie_space_id`, `endpoint_name`, `uc_function_name`, or `connection_name`. + +To find a KA tile_id, use `manage_ka(action="find_by_name", name="Your KA Name")`. +To find a Genie space_id, use `find_genie_by_name(display_name="Your Genie Name")`. + +### Writing Good Descriptions + +The `description` field is critical for routing. Make it specific: + +**Good descriptions:** +- "Handles billing questions including invoices, payments, refunds, and subscription changes" +- "Answers technical questions about API errors, integration issues, and product bugs" +- "Provides information about HR policies, PTO, benefits, and employee handbook" + +**Bad descriptions:** +- "Billing agent" (too vague) +- "Handles stuff" (not helpful) +- "Technical" (not specific) + +## Provisioning Timeline + +After creation, the Supervisor Agent endpoint needs to provision: + +| Status | Meaning | Duration | +|--------|---------|----------| +| `PROVISIONING` | Creating the supervisor | 2-5 minutes | +| `ONLINE` | Ready to route queries | - | +| `OFFLINE` | Not currently running | - | + +Use `manage_mas` with `action="get"` to check the status. + +## Adding Example Questions + +Example questions help with evaluation and can guide routing optimization: + +```json +{ + "examples": [ + { + "question": "I haven't received my invoice for this month", + "guideline": "Should be routed to billing_agent" + }, + { + "question": "The API is returning a 500 error", + "guideline": "Should be routed to technical_agent" + }, + { + "question": "How many vacation days do I have?", + "guideline": "Should be routed to hr_agent" + } + ] +} +``` + +If the Supervisor Agent is not yet `ONLINE`, examples are queued and added automatically when ready. + +## Best Practices + +### Agent Design + +1. **Specialized agents**: Each agent should have a clear, distinct purpose +2. **Non-overlapping domains**: Avoid agents with similar descriptions +3. **Clear boundaries**: Define what each agent does and doesn't handle + +### Instructions + +Provide routing instructions: + +``` +You are a customer support supervisor. Your job is to route user queries to the right specialist: + +1. For billing, payments, or subscription questions → billing_agent +2. For technical issues, bugs, or API problems → technical_agent +3. For HR, benefits, or policy questions → hr_agent + +If the query is unclear or spans multiple domains, ask the user to clarify. +``` + +### Fallback Handling + +Consider adding a general-purpose agent for queries that don't fit elsewhere: + +```json +{ + "name": "general_agent", + "endpoint_name": "general-support-endpoint", + "description": "Handles general inquiries that don't fit other categories, provides navigation help" +} +``` + +## Example Workflow + +1. **Deploy specialized agents** as model serving endpoints: + - `billing-assistant-endpoint` + - `tech-support-endpoint` + - `hr-assistant-endpoint` + +2. **Create the MAS**: + - Configure agents with clear descriptions + - Add routing instructions + +3. **Wait for ONLINE status** (2-5 minutes) + +4. **Add example questions** for evaluation + +5. **Test routing** with various query types + +## Updating a Supervisor Agent + +To update an existing Supervisor Agent: + +1. **Add/remove agents**: Call `manage_mas` with `action="create_or_update"` and updated `agents` list +2. **Update descriptions**: Change agent descriptions to improve routing +3. **Modify instructions**: Update routing rules + +The tool finds the existing Supervisor Agent by name and updates it. + +## Troubleshooting + +### Queries routed to wrong agent + +- Review and improve agent descriptions +- Make descriptions more specific and distinct +- Add examples that demonstrate correct routing + +### Endpoint not responding + +- Verify each underlying model serving endpoint is running +- Check endpoint logs for errors +- Ensure endpoints accept the expected input format + +### Slow responses + +- Check latency of underlying endpoints +- Consider endpoint scaling settings +- Monitor for cold start issues + +## Advanced: Hierarchical Routing + +For complex scenarios, you can create multiple levels of Supervisor Agents: + +``` +Top-level Supervisor +├── Customer Support Supervisor +│ ├── billing_agent +│ ├── technical_agent +│ └── general_agent +├── Sales Supervisor +│ ├── pricing_agent +│ ├── demo_agent +│ └── contract_agent +└── Internal Supervisor + ├── hr_agent + └── it_helpdesk_agent +``` + +Each sub-supervisor is deployed as an endpoint and configured as an agent in the top-level supervisor. diff --git a/.claude/skills/databricks-agent-bricks/SKILL.md b/.claude/skills/databricks-agent-bricks/SKILL.md new file mode 100644 index 00000000..4aff7acb --- /dev/null +++ b/.claude/skills/databricks-agent-bricks/SKILL.md @@ -0,0 +1,211 @@ +--- +name: databricks-agent-bricks +description: "Create and manage Databricks Agent Bricks: Knowledge Assistants (KA) for document Q&A, Genie Spaces for SQL exploration, and Supervisor Agents (MAS) for multi-agent orchestration. Use when building conversational AI applications on Databricks." +--- + +# Agent Bricks + +Create and manage Databricks Agent Bricks - pre-built AI components for building conversational applications. + +## Overview + +Agent Bricks are three types of pre-built AI tiles in Databricks: + +| Brick | Purpose | Data Source | +|-------|---------|-------------| +| **Knowledge Assistant (KA)** | Document-based Q&A using RAG | PDF/text files in Volumes | +| **Genie Space** | Natural language to SQL | Unity Catalog tables | +| **Supervisor Agent (MAS)** | Multi-agent orchestration | Model serving endpoints | + +## Prerequisites + +Before creating Agent Bricks, ensure you have the required data: + +### For Knowledge Assistants +- **Documents in a Volume**: PDF, text, or other files stored in a Unity Catalog volume +- Generate synthetic documents using the `databricks-unstructured-pdf-generation` skill if needed + +### For Genie Spaces +- **See the `databricks-genie` skill** for comprehensive Genie Space guidance +- Tables in Unity Catalog with the data to explore +- Generate raw data using the `databricks-synthetic-data-generation` skill +- Create tables using the `databricks-spark-declarative-pipelines` skill + +### For Supervisor Agents +- **Model Serving Endpoints**: Deployed agent endpoints (KA endpoints, custom agents, fine-tuned models) +- **Genie Spaces**: Existing Genie spaces can be used directly as agents for SQL-based queries +- Mix and match endpoint-based and Genie-based agents in the same Supervisor Agent + +### For Unity Catalog Functions +- **Existing UC Function**: Function already registered in Unity Catalog +- Agent service principal has `EXECUTE` privilege on the function + +### For External MCP Servers +- **Existing UC HTTP Connection**: Connection configured with `is_mcp_connection: 'true'` +- Agent service principal has `USE CONNECTION` privilege on the connection + +## MCP Tools + +### Knowledge Assistant Tool + +**manage_ka** - Manage Knowledge Assistants (KA) +- `action`: "create_or_update", "get", "find_by_name", or "delete" +- `name`: Name for the KA (for create_or_update, find_by_name) +- `volume_path`: Path to documents (e.g., `/Volumes/catalog/schema/volume/folder`) (for create_or_update) +- `description`: (optional) What the KA does (for create_or_update) +- `instructions`: (optional) How the KA should answer (for create_or_update) +- `tile_id`: The KA tile ID (for get, delete, or update via create_or_update) +- `add_examples_from_volume`: (optional, default: true) Auto-add examples from JSON files (for create_or_update) + +Actions: +- **create_or_update**: Requires `name`, `volume_path`. Optionally pass `tile_id` to update. +- **get**: Requires `tile_id`. Returns tile_id, name, description, endpoint_status, knowledge_sources, examples_count. +- **find_by_name**: Requires `name` (exact match). Returns found, tile_id, name, endpoint_name, endpoint_status. Use this to look up an existing KA when you know the name but not the tile_id. +- **delete**: Requires `tile_id`. + +### Genie Space Tools + +**For comprehensive Genie guidance, use the `databricks-genie` skill.** + +Basic tools available: + +- `create_or_update_genie` - Create or update a Genie Space +- `get_genie` - Get Genie Space details +- `delete_genie` - Delete a Genie Space + +See `databricks-genie` skill for: +- Table inspection workflow +- Sample question best practices +- Curation (instructions, certified queries) + +**IMPORTANT**: There is NO system table for Genie spaces (e.g., `system.ai.genie_spaces` does not exist). To find a Genie space by name, use the `find_genie_by_name` tool. + +### Supervisor Agent Tool + +**manage_mas** - Manage Supervisor Agents (MAS) +- `action`: "create_or_update", "get", "find_by_name", or "delete" +- `name`: Name for the Supervisor Agent (for create_or_update, find_by_name) +- `agents`: List of agent configurations (for create_or_update), each with: + - `name`: Agent identifier (required) + - `description`: What this agent handles - critical for routing (required) + - `ka_tile_id`: Knowledge Assistant tile ID (use for document Q&A agents - recommended for KAs) + - `genie_space_id`: Genie space ID (use for SQL-based data agents) + - `endpoint_name`: Model serving endpoint name (for custom agents) + - `uc_function_name`: Unity Catalog function name in format `catalog.schema.function_name` + - `connection_name`: Unity Catalog connection name (for external MCP servers) + - Note: Provide exactly one of: `ka_tile_id`, `genie_space_id`, `endpoint_name`, `uc_function_name`, or `connection_name` +- `description`: (optional) What the Supervisor Agent does (for create_or_update) +- `instructions`: (optional) Routing instructions for the supervisor (for create_or_update) +- `tile_id`: The Supervisor Agent tile ID (for get, delete, or update via create_or_update) +- `examples`: (optional) List of example questions with `question` and `guideline` fields (for create_or_update) + +Actions: +- **create_or_update**: Requires `name`, `agents`. Optionally pass `tile_id` to update. +- **get**: Requires `tile_id`. Returns tile_id, name, description, endpoint_status, agents, examples_count. +- **find_by_name**: Requires `name` (exact match). Returns found, tile_id, name, endpoint_status, agents_count. Use this to look up an existing Supervisor Agent when you know the name but not the tile_id. +- **delete**: Requires `tile_id`. + +## Typical Workflow + +### 1. Generate Source Data + +Before creating Agent Bricks, generate the required source data: + +**For KA (document Q&A)**: +``` +1. Use `databricks-unstructured-pdf-generation` skill to generate PDFs +2. PDFs are saved to a Volume with companion JSON files (question/guideline pairs) +``` + +**For Genie (SQL exploration)**: +``` +1. Use `databricks-synthetic-data-generation` skill to create raw parquet data +2. Use `databricks-spark-declarative-pipelines` skill to create bronze/silver/gold tables +``` + +### 2. Create the Agent Brick + +Use `manage_ka(action="create_or_update", ...)` or `manage_mas(action="create_or_update", ...)` with your data sources. + +### 3. Wait for Provisioning + +Newly created KA and MAS tiles need time to provision. The endpoint status will progress: +- `PROVISIONING` - Being created (can take 2-5 minutes) +- `ONLINE` - Ready to use +- `OFFLINE` - Not running + +### 4. Add Examples (Automatic) + +For KA, if `add_examples_from_volume=true`, examples are automatically extracted from JSON files in the volume and added once the endpoint is `ONLINE`. + +## Best Practices + +1. **Use meaningful names**: Names are sanitized automatically (spaces become underscores) +2. **Provide descriptions**: Helps users understand what the brick does +3. **Add instructions**: Guide the AI's behavior and tone +4. **Include sample questions**: Shows users how to interact with the brick +5. **Use the workflow**: Generate data first, then create the brick + +## Example: Multi-Modal Supervisor Agent + +```python +manage_mas( + action="create_or_update", + name="Enterprise Support Supervisor", + agents=[ + { + "name": "knowledge_base", + "ka_tile_id": "f32c5f73-466b-...", + "description": "Answers questions about company policies, procedures, and documentation from indexed files" + }, + { + "name": "analytics_engine", + "genie_space_id": "01abc123...", + "description": "Runs SQL analytics on usage metrics, performance stats, and operational data" + }, + { + "name": "ml_classifier", + "endpoint_name": "custom-classification-endpoint", + "description": "Classifies support tickets and predicts resolution time using custom ML model" + }, + { + "name": "data_enrichment", + "uc_function_name": "support.utils.enrich_ticket_data", + "description": "Enriches support ticket data with customer history and context" + }, + { + "name": "ticket_operations", + "connection_name": "ticket_system_mcp", + "description": "Creates, updates, assigns, and closes support tickets in external ticketing system" + } + ], + description="Comprehensive enterprise support agent with knowledge retrieval, analytics, ML, data enrichment, and ticketing operations", + instructions=""" + Route queries as follows: + 1. Policy/procedure questions → knowledge_base + 2. Data analysis requests → analytics_engine + 3. Ticket classification → ml_classifier + 4. Customer context lookups → data_enrichment + 5. Ticket creation/updates → ticket_operations + + If a query spans multiple domains, chain agents: + - First gather information (analytics_engine or knowledge_base) + - Then take action (ticket_operations) + """ +) +``` + +## Related Skills + +- **[databricks-genie](../databricks-genie/SKILL.md)** - Comprehensive Genie Space creation, curation, and Conversation API guidance +- **[databricks-unstructured-pdf-generation](../databricks-unstructured-pdf-generation/SKILL.md)** - Generate synthetic PDFs to feed into Knowledge Assistants +- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Create raw data for Genie Space tables +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Build bronze/silver/gold tables consumed by Genie Spaces +- **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - Deploy custom agent endpoints used as MAS agents +- **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - Build vector indexes for RAG applications paired with KAs + +## See Also + +- `1-knowledge-assistants.md` - Detailed KA patterns and examples +- `databricks-genie` skill - Detailed Genie patterns, curation, and examples +- `2-supervisor-agents.md` - Detailed MAS patterns and examples diff --git a/.claude/skills/aibi-dashboards/SKILL.md b/.claude/skills/databricks-aibi-dashboards/SKILL.md similarity index 98% rename from .claude/skills/aibi-dashboards/SKILL.md rename to .claude/skills/databricks-aibi-dashboards/SKILL.md index bb846b05..41dbeec6 100644 --- a/.claude/skills/aibi-dashboards/SKILL.md +++ b/.claude/skills/databricks-aibi-dashboards/SKILL.md @@ -1,6 +1,6 @@ --- -name: aibi-dashboards -description: "Create AI/BI dashboards. CRITICAL: You MUST test ALL SQL queries via execute_sql BEFORE deploying. Follow guidelines strictly." +name: databricks-aibi-dashboards +description: "Create Databricks AI/BI dashboards. CRITICAL: You MUST test ALL SQL queries via execute_sql BEFORE deploying. Follow guidelines strictly." --- # AI/BI Dashboard Skill @@ -915,3 +915,9 @@ print(result["url"]) - Multiple items in the `lines` array are **concatenated**, not displayed on separate lines - Use **separate text widgets** for title and subtitle at different y positions - Example: title at y=0 with height=1, subtitle at y=1 with height=1 + +## Related Skills + +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for querying the underlying data and system tables +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - for building the data pipelines that feed dashboards +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - for scheduling dashboard data refreshes diff --git a/.claude/skills/databricks-app-apx/SKILL.md b/.claude/skills/databricks-app-apx/SKILL.md index 54c2767d..2ee96baa 100644 --- a/.claude/skills/databricks-app-apx/SKILL.md +++ b/.claude/skills/databricks-app-apx/SKILL.md @@ -157,7 +157,7 @@ Manually verify in browser: ### Deploy to Databricks -Use DABs to deploy your APX application to Databricks. See the `asset-bundles` skill for complete deployment guidance. +Use DABs to deploy your APX application to Databricks. See the `databricks-asset-bundles` skill for complete deployment guidance. ### Monitor Application Logs @@ -244,3 +244,10 @@ Create two markdown files: - **[best-practices.md](best-practices.md)** - Best practices, anti-patterns, debugging Read these files only when actively writing that type of code or debugging issues. + +## Related Skills + +- **[databricks-app-python](../databricks-app-python/SKILL.md)** - for Streamlit, Dash, Gradio, or Flask apps +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying APX apps via DABs +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - backend SDK integration +- **[databricks-lakebase-provisioned](../databricks-lakebase-provisioned/SKILL.md)** - adding persistent PostgreSQL state to apps diff --git a/.claude/skills/databricks-app-python/1-authorization.md b/.claude/skills/databricks-app-python/1-authorization.md new file mode 100644 index 00000000..0a84f629 --- /dev/null +++ b/.claude/skills/databricks-app-python/1-authorization.md @@ -0,0 +1,150 @@ +# Authorization for Databricks Apps + +Databricks Apps supports two complementary authorization models. Use one or both depending on your app's needs. + +**Docs**: https://docs.databricks.com/aws/en/dev-tools/databricks-apps/auth + +--- + +## App Authorization (Service Principal) + +Each app gets a dedicated service principal. Databricks auto-injects credentials: + +- `DATABRICKS_CLIENT_ID` — OAuth client ID +- `DATABRICKS_CLIENT_SECRET` — OAuth client secret + +**You don't need to read these manually.** The SDK `Config()` detects them automatically: + +```python +from databricks.sdk.core import Config +from databricks import sql + +cfg = Config() # Auto-detects SP credentials from environment +conn = sql.connect( + server_hostname=cfg.host, + http_path="/sql/1.0/warehouses/", + credentials_provider=lambda: cfg.authenticate, +) +``` + +**Use for**: background tasks, shared data access, logging, external service calls. + +**Limitation**: all users share the same permissions — no per-user access control. + +--- + +## User Authorization (On-Behalf-Of) + +Allows the app to act with the identity of the current user. Databricks forwards the user's access token to the app via HTTP header. + +**Use for**: user-specific data queries, Unity Catalog row/column filters, audit trails. + +**Prerequisite**: workspace admin must enable user authorization (Public Preview). Add scopes when creating/editing the app in the UI. + +### Retrieving the User Token Per Framework + +```python +# Streamlit +import streamlit as st +user_token = st.context.headers.get("x-forwarded-access-token") + +# Dash / Flask +from flask import request +user_token = request.headers.get("x-forwarded-access-token") + +# Gradio +import gradio as gr +def handler(message, request: gr.Request): + user_token = request.headers.get("x-forwarded-access-token") + +# FastAPI +from fastapi import Request +async def endpoint(request: Request): + user_token = request.headers.get("x-forwarded-access-token") + +# Reflex +user_token = session.http_conn.headers.get("x-forwarded-access-token") +``` + +### Querying with User Token + +```python +from databricks.sdk.core import Config +from databricks import sql + +cfg = Config() +user_token = get_user_token() # Per-framework method above + +conn = sql.connect( + server_hostname=cfg.host, + http_path="/sql/1.0/warehouses/", + access_token=user_token, # User's token, not SP credentials +) +``` + +--- + +## Combining Both Models + +Use app auth for shared operations and user auth for user-specific data: + +```python +from databricks.sdk.core import Config +from databricks import sql + +cfg = Config() + +def get_app_connection(warehouse_http_path: str): + """App auth — shared data, logging, background tasks.""" + return sql.connect( + server_hostname=cfg.host, + http_path=warehouse_http_path, + credentials_provider=lambda: cfg.authenticate, + ) + +def get_user_connection(warehouse_http_path: str, user_token: str): + """User auth — respects Unity Catalog row/column filters.""" + return sql.connect( + server_hostname=cfg.host, + http_path=warehouse_http_path, + access_token=user_token, + ) +``` + +--- + +## OAuth Scopes + +When adding user authorization, select only the scopes your app needs: + +| Scope | Grants Access To | +|-------|-----------------| +| `sql` | SQL warehouse queries | +| `files.files` | Files and directories | +| `dashboards.genie` | Genie spaces | +| `iam.access-control:read` | Access control (default) | +| `iam.current-user:read` | Current user identity (default) | + +**Best practice**: request minimum required scopes. Databricks blocks access outside approved scopes even if the user has broader permissions. + +--- + +## When to Use Which + +| Scenario | Model | +|----------|-------| +| All users see same data | App auth only | +| User-specific row/column filters | User auth | +| Background jobs, logging | App auth | +| Audit trail per user | User auth | +| Mixed shared + personal data | Both | + +--- + +## Best Practices + +- Never log, print, or write tokens to files +- Grant service principal minimum required permissions on resources +- Use `CAN MANAGE` only for trusted developers; `CAN USE` for app users +- Enforce peer review for app code before production deployment +- Cookbook auth examples: [Streamlit](https://apps-cookbook.dev/docs/streamlit/authentication/users_get_current) · [Dash](https://apps-cookbook.dev/docs/dash/authentication/users_get_current) · [Reflex](https://apps-cookbook.dev/docs/reflex/authentication/users_get_current) diff --git a/.claude/skills/databricks-app-python/2-app-resources.md b/.claude/skills/databricks-app-python/2-app-resources.md new file mode 100644 index 00000000..dd911c4b --- /dev/null +++ b/.claude/skills/databricks-app-python/2-app-resources.md @@ -0,0 +1,120 @@ +# App Resources and Communication Strategies + +Databricks Apps integrate with platform resources via managed connections. Use resources instead of hardcoding IDs for portability and security. + +**Docs**: https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources + +--- + +## Supported Resource Types + +| Resource | Default Key | Permissions | Use Case | +|----------|-------------|-------------|----------| +| SQL warehouse | `sql-warehouse` | Can use, Can manage | Querying Delta tables | +| Lakebase database | `database` | Can connect and create | Low-latency transactional data | +| Model serving endpoint | `serving-endpoint` | Can view, Can query, Can manage | AI/ML inference | +| Secret | `secret` | Can read, Can write, Can manage | API keys, tokens | +| Unity Catalog volume | `volume` | Can read, Can read and write | File storage | +| Vector search index | `vector-search-index` | Can select | Semantic search | +| Genie space | `genie-space` | Can view, Can run, Can edit | Natural language analytics | +| UC connection | `connection` | Use Connection | External data sources | +| UC function | `function` | Can execute | SQL/Python functions | +| MLflow experiment | `experiment` | Can read, Can edit | ML experiment tracking | +| Lakeflow job | `job` | Can view, Can manage run | Data pipelines | + +--- + +## Configuring Resources in app.yaml + +Use `valueFrom` to reference resources — never hardcode IDs: + +```yaml +env: + - name: DATABRICKS_WAREHOUSE_ID + valueFrom: sql-warehouse + + - name: SERVING_ENDPOINT_NAME + valueFrom: serving-endpoint + + - name: DB_CONNECTION_STRING + valueFrom: database +``` + +Add resources via the Databricks Apps UI when creating or editing an app: +1. Navigate to Configure step +2. Click **+ Add resource** +3. Select resource type and set permissions +4. Assign a key (referenced in `valueFrom`) + +--- + +## Communication Strategies + +Choose your data backend based on access pattern: + +| Strategy | When to Use | Library | Connection Pattern | +|----------|-------------|---------|-------------------| +| **SQL Warehouse** | Analytical queries on Delta tables | `databricks-sql-connector` | `sql.connect()` with `Config()` | +| **Lakebase (PostgreSQL)** | Low-latency transactional CRUD | `psycopg2` / `asyncpg` | Standard PostgreSQL via auto-injected env vars | +| **Databricks SDK** | Platform API calls (jobs, clusters, UC) | `databricks-sdk` | `WorkspaceClient()` | +| **Model Serving** | AI/ML inference requests | `requests` or SDK | REST call to serving endpoint | +| **Unity Catalog Functions** | Server-side compute (SQL/Python UDFs) | `databricks-sql-connector` | Execute via SQL warehouse | + +### SQL Warehouse Pattern + +```python +import os +from databricks.sdk.core import Config +from databricks import sql + +cfg = Config() +conn = sql.connect( + server_hostname=cfg.host, + http_path=f"/sql/1.0/warehouses/{os.getenv('DATABRICKS_WAREHOUSE_ID')}", + credentials_provider=lambda: cfg.authenticate, +) + +with conn.cursor() as cursor: + cursor.execute("SELECT * FROM catalog.schema.table LIMIT 100") + rows = cursor.fetchall() +``` + +### Model Serving Pattern + +```python +import os, requests +from databricks.sdk.core import Config + +cfg = Config() +headers = cfg.authenticate() +headers["Content-Type"] = "application/json" + +endpoint = os.getenv("SERVING_ENDPOINT_NAME") +response = requests.post( + f"https://{cfg.host}/serving-endpoints/{endpoint}/invocations", + headers=headers, + json={"inputs": [{"prompt": "Hello"}]}, +) +result = response.json() +``` + +### SDK Pattern + +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() # Auto-detects credentials +for cluster in w.clusters.list(): + print(f"{cluster.cluster_name}: {cluster.state}") +``` + +For Lakebase patterns, see [5-lakebase.md](5-lakebase.md). + +--- + +## Best Practices + +- Always use `valueFrom` — keeps apps portable between environments +- Grant service principal minimum required permissions (e.g., `CAN USE` not `CAN MANAGE` for SQL warehouse) +- Use Lakebase for transactional workloads; SQL warehouse for analytical workloads +- For external services, use UC connections or secrets (never hardcode API keys) diff --git a/.claude/skills/databricks-app-python/3-frameworks.md b/.claude/skills/databricks-app-python/3-frameworks.md new file mode 100644 index 00000000..cb1ef87e --- /dev/null +++ b/.claude/skills/databricks-app-python/3-frameworks.md @@ -0,0 +1,246 @@ +# Supported Frameworks + +All frameworks below are **pre-installed** in the Databricks Apps runtime. Claude already knows how to use them — this guide covers only **Databricks-specific** patterns. For full examples and recipes, see the **[Databricks Apps Cookbook](https://apps-cookbook.dev/)**. + +--- + +## Dash + +**Best for**: Production dashboards, BI tools, complex interactive visualizations. + +**Critical**: Always use `dash-bootstrap-components` for layout and styling. + +```python +import dash +import dash_bootstrap_components as dbc + +app = dash.Dash( + __name__, + external_stylesheets=[dbc.themes.BOOTSTRAP, dbc.icons.FONT_AWESOME], + title="My Dashboard", +) +``` + +| Detail | Value | +|--------|-------| +| Pre-installed version | 2.18.1 | +| app.yaml command | `["python", "app.py"]` | +| Default port | 8050 (set `DATABRICKS_APP_PORT=8080` or use `app.run(port=8080)`) | +| Auth header | `request.headers.get('x-forwarded-access-token')` (Flask under the hood) | + +**Databricks tips**: +- Use `dbc.themes.BOOTSTRAP` and `dbc.icons.FONT_AWESOME` for consistent styling +- Use Bootstrap badge color names (`"success"`, `"danger"`), not hex colors, for `dbc.Badge` +- Use `prevent_initial_call=True` on expensive callbacks +- Use `dcc.Store` for client-side caching + +**Cookbook**: [apps-cookbook.dev/docs/category/dash](https://apps-cookbook.dev/docs/category/dash) — tables, volumes, AI/ML, workflows, dashboards, compute, auth, external services. + +--- + +## Streamlit + +**Best for**: Rapid prototyping, data science apps, internal tools, notebook-to-app workflow. + +**Critical**: Always use `@st.cache_resource` for database connections. + +```python +import streamlit as st +from databricks.sdk.core import Config +from databricks import sql + +st.set_page_config(page_title="My App", layout="wide") # Must be first! + +@st.cache_resource(ttl=300) +def get_connection(): + cfg = Config() + return sql.connect( + server_hostname=cfg.host, + http_path="/sql/1.0/warehouses/", + credentials_provider=lambda: cfg.authenticate, + ) +``` + +| Detail | Value | +|--------|-------| +| Pre-installed version | 1.38.0 | +| app.yaml command | `["streamlit", "run", "app.py"]` | +| Auth header | `st.context.headers.get('x-forwarded-access-token')` | + +**Databricks tips**: +- `st.set_page_config()` must be the **first** Streamlit command +- `@st.cache_resource` for connections/models; `@st.cache_data(ttl=...)` for query results +- Use `st.form()` to batch inputs and prevent reruns on every keystroke +- Use `st.column_config` for formatted DataFrames (currency, dates) + +**Cookbook**: [apps-cookbook.dev/docs/category/streamlit](https://apps-cookbook.dev/docs/category/streamlit) — tables, volumes, AI/ML, workflows, visualizations, dashboards, compute, auth, external services. + +--- + +## Gradio + +**Best for**: ML model demos, chat interfaces, image/audio/video processing UIs. + +**Critical**: Use `gr.Request` parameter to access auth headers. + +```python +import gradio as gr +import requests +from databricks.sdk.core import Config + +cfg = Config() + +def predict(message, request: gr.Request): + user_token = request.headers.get("x-forwarded-access-token") + # Query model serving endpoint + headers = {**cfg.authenticate(), "Content-Type": "application/json"} + resp = requests.post( + f"https://{cfg.host}/serving-endpoints/my-model/invocations", + headers=headers, + json={"inputs": [{"prompt": message}]}, + ) + return resp.json()["predictions"][0] + +demo = gr.Interface(fn=predict, inputs="text", outputs="text") +demo.launch(server_name="0.0.0.0", server_port=8080) +``` + +| Detail | Value | +|--------|-------| +| Pre-installed version | 4.44.0 | +| app.yaml command | `["python", "app.py"]` | +| Default port | 7860 (override with `server_port=8080` or `GRADIO_SERVER_PORT=8080`) | +| Auth header | `request.headers.get('x-forwarded-access-token')` via `gr.Request` | + +**Databricks tips**: +- Natural fit for model serving endpoint integration +- Use `gr.ChatInterface` for conversational AI demos +- Use `gr.Blocks` for complex multi-component layouts + +**Docs**: [gradio.app/docs](https://www.gradio.app/docs) + +--- + +## Flask + +**Best for**: Custom REST APIs, lightweight web apps, webhook receivers. + +**Critical**: Deploy with Gunicorn — never use Flask's dev server in production. + +```python +from flask import Flask, request, jsonify +from databricks.sdk.core import Config +from databricks import sql + +app = Flask(__name__) +cfg = Config() + +@app.route("/api/data") +def get_data(): + conn = sql.connect( + server_hostname=cfg.host, + http_path="/sql/1.0/warehouses/", + credentials_provider=lambda: cfg.authenticate, + ) + with conn.cursor() as cursor: + cursor.execute("SELECT * FROM catalog.schema.table LIMIT 10") + return jsonify(cursor.fetchall()) +``` + +| Detail | Value | +|--------|-------| +| Pre-installed version | 3.0.3 | +| app.yaml command | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8080"]` | +| Auth header | `request.headers.get('x-forwarded-access-token')` | + +**Databricks tips**: +- Use connection pooling (Flask doesn't cache connections like Streamlit) +- Gunicorn workers (`-w 4`) handle concurrent requests +- Use `request.headers` for user authorization tokens + +--- + +## FastAPI + +**Best for**: Modern async APIs, auto-generated OpenAPI/Swagger docs, high-performance backends. + +**Critical**: Deploy with uvicorn. + +```python +from fastapi import FastAPI, Request +from databricks.sdk.core import Config +from databricks import sql + +app = FastAPI(title="My API") +cfg = Config() + +@app.get("/api/data") +async def get_data(request: Request): + user_token = request.headers.get("x-forwarded-access-token") + conn = sql.connect( + server_hostname=cfg.host, + http_path="/sql/1.0/warehouses/", + access_token=user_token, + ) + with conn.cursor() as cursor: + cursor.execute("SELECT * FROM catalog.schema.table LIMIT 10") + return cursor.fetchall() +``` + +| Detail | Value | +|--------|-------| +| Pre-installed version | 0.115.0 | +| app.yaml command | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]` | +| Auth header | `request.headers.get('x-forwarded-access-token')` via `Request` | + +**Databricks tips**: +- Auto-generates OpenAPI docs at `/docs` (Swagger) and `/redoc` +- Databricks SQL connector is synchronous — use `asyncio.to_thread()` for async endpoints +- Good choice for API backends that serve APX (FastAPI + React) apps + +**Cookbook**: [apps-cookbook.dev/docs/category/fastapi](https://apps-cookbook.dev/docs/category/fastapi) — getting started, endpoint examples. + +--- + +## Reflex + +**Best for**: Full-stack Python apps with reactive UIs, no JavaScript required. + +```python +import reflex as rx +from databricks.sdk.core import Config + +cfg = Config() + +class State(rx.State): + data: list[dict] = [] + + def load_data(self): + from databricks import sql + conn = sql.connect( + server_hostname=cfg.host, + http_path="/sql/1.0/warehouses/", + credentials_provider=lambda: cfg.authenticate, + ) + with conn.cursor() as cursor: + cursor.execute("SELECT * FROM catalog.schema.table LIMIT 10") + self.data = [dict(zip([d[0] for d in cursor.description], row)) for row in cursor.fetchall()] +``` + +| Detail | Value | +|--------|-------| +| app.yaml command | `["reflex", "run", "--env", "prod"]` | +| Auth header | `session.http_conn.headers.get('x-forwarded-access-token')` | + +**Cookbook**: [apps-cookbook.dev/docs/category/reflex](https://apps-cookbook.dev/docs/category/reflex) — tables, volumes, AI/ML, workflows, dashboards, compute, auth, external services. + +--- + +## Common: All Frameworks + +- All frameworks are **pre-installed** — no need to add them to `requirements.txt` +- Add only additional packages your app needs to `requirements.txt` +- SDK `Config()` auto-detects credentials from injected environment variables +- Databricks Apps expects apps to listen on **port 8080** (configure your framework accordingly) +- For framework-specific deployment commands, see [4-deployment.md](4-deployment.md) +- For authorization integration, see [1-authorization.md](1-authorization.md) diff --git a/.claude/skills/databricks-app-python/4-deployment.md b/.claude/skills/databricks-app-python/4-deployment.md new file mode 100644 index 00000000..688f1f21 --- /dev/null +++ b/.claude/skills/databricks-app-python/4-deployment.md @@ -0,0 +1,142 @@ +# Deploying Databricks Apps + +Three deployment options: Databricks CLI (simplest), Asset Bundles (multi-environment), or MCP tools (programmatic). + +**Cookbook deployment guide**: https://apps-cookbook.dev/docs/deploy + +--- + +## Option 1: Databricks CLI + +**Best for**: quick deployments, single environment. + +### Step 1: Create app.yaml + +```yaml +command: + - "python" # Adjust per framework — see table below + - "app.py" + +env: + - name: DATABRICKS_WAREHOUSE_ID + valueFrom: sql-warehouse + - name: USE_MOCK_BACKEND + value: "false" +``` + +### app.yaml Commands Per Framework + +| Framework | Command | +|-----------|---------| +| Dash | `["python", "app.py"]` | +| Streamlit | `["streamlit", "run", "app.py"]` | +| Gradio | `["python", "app.py"]` | +| Flask | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8080"]` | +| FastAPI | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]` | +| Reflex | `["reflex", "run", "--env", "prod"]` | + +### Step 2: Create and Deploy + +```bash +# Create the app +databricks apps create + +# Upload source code +databricks workspace mkdirs /Workspace/Users//apps/ +databricks workspace import-dir . /Workspace/Users//apps/ + +# Deploy +databricks apps deploy \ + --source-code-path /Workspace/Users//apps/ + +# Add resources via UI (SQL warehouse, Lakebase, etc.) + +# Check status and URL +databricks apps get +``` + +### Redeployment + +```bash +databricks workspace delete /Workspace/Users//apps/ --recursive +databricks workspace import-dir . /Workspace/Users//apps/ +databricks apps deploy \ + --source-code-path /Workspace/Users//apps/ +``` + +--- + +## Option 2: Databricks Asset Bundles (DABs) + +**Best for**: multi-environment deployments (dev/staging/prod), version-controlled infrastructure. + +**Recommended workflow**: deploy via CLI first to validate, then generate bundle config. + +### Generate Bundle from Existing App + +```bash +databricks bundle generate app \ + --existing-app-name \ + --key +``` + +This creates: +- `resources/.app.yml` — app resource definition +- `src/app/` — app source files including `app.yaml` + +### Deploy with Bundles + +```bash +# Validate +databricks bundle validate -t dev + +# Deploy +databricks bundle deploy -t dev + +# Start the app (required after deployment) +databricks bundle run -t dev + +# Production +databricks bundle deploy -t prod +databricks bundle run -t prod +``` + +**Key difference from other resources**: environment variables go in `src/app/app.yaml`, not `databricks.yml`. + +For complete DABs guidance, use the **databricks-asset-bundles** skill. + +--- + +## Option 3: MCP Tools + +For programmatic app lifecycle management, see [6-mcp-approach.md](6-mcp-approach.md). + +--- + +## Post-Deployment + +### Check Logs + +```bash +databricks apps logs +``` + +**Key patterns in logs**: +- `[SYSTEM]` — deployment status, file updates, dependency installation +- `[APP]` — application output, framework messages +- `Deployment successful` — app deployed correctly +- `App started successfully` — app is running +- `Error:` — check stack traces + +### Verify + +1. Access app URL (from `databricks apps get `) +2. Check all pages load correctly +3. Verify data connectivity (look for backend initialization messages in logs) +4. Test user authorization flow if enabled + +### Configure Permissions + +- Set `CAN USE` for approved users/groups +- Set `CAN MANAGE` only for trusted developers +- Verify service principal has required resource permissions diff --git a/.claude/skills/databricks-app-python/5-lakebase.md b/.claude/skills/databricks-app-python/5-lakebase.md new file mode 100644 index 00000000..c6615609 --- /dev/null +++ b/.claude/skills/databricks-app-python/5-lakebase.md @@ -0,0 +1,141 @@ +# Lakebase (PostgreSQL) Connectivity + +Lakebase provides low-latency transactional storage for Databricks Apps via a managed PostgreSQL interface. + +**Docs**: https://docs.databricks.com/aws/en/dev-tools/databricks-apps/lakebase + +--- + +## When to Use Lakebase + +| Use Case | Recommended Backend | +|----------|-------------------| +| Analytical queries on Delta tables | SQL Warehouse | +| Low-latency transactional CRUD | **Lakebase** | +| App-specific metadata/config | **Lakebase** | +| User session data | **Lakebase** | +| Large-scale data exploration | SQL Warehouse | + +--- + +## Setup + +1. Add Lakebase as an app resource in the Databricks UI (resource type: **Lakebase database**) +2. Databricks auto-injects PostgreSQL connection env vars: + +| Variable | Description | +|----------|-------------| +| `PGHOST` | Database hostname | +| `PGDATABASE` | Database name | +| `PGUSER` | PostgreSQL role (created per app) | +| `PGPASSWORD` | Role password | +| `PGPORT` | Port (typically 5432) | + +3. Reference in `app.yaml`: + +```yaml +env: + - name: DB_CONNECTION_STRING + valueFrom: + resource: database +``` + +--- + +## Connection Patterns + +### psycopg2 (Synchronous) + +```python +import os +import psycopg2 + +conn = psycopg2.connect( + host=os.getenv("PGHOST"), + database=os.getenv("PGDATABASE"), + user=os.getenv("PGUSER"), + password=os.getenv("PGPASSWORD"), + port=os.getenv("PGPORT", "5432"), +) + +with conn.cursor() as cur: + cur.execute("SELECT * FROM my_table LIMIT 10") + rows = cur.fetchall() + +conn.close() +``` + +### asyncpg (Asynchronous) + +```python +import os +import asyncpg + +async def get_data(): + conn = await asyncpg.connect( + host=os.getenv("PGHOST"), + database=os.getenv("PGDATABASE"), + user=os.getenv("PGUSER"), + password=os.getenv("PGPASSWORD"), + port=int(os.getenv("PGPORT", "5432")), + ) + rows = await conn.fetch("SELECT * FROM my_table LIMIT 10") + await conn.close() + return rows +``` + +### SQLAlchemy + +```python +import os +from sqlalchemy import create_engine + +DATABASE_URL = ( + f"postgresql://{os.getenv('PGUSER')}:{os.getenv('PGPASSWORD')}" + f"@{os.getenv('PGHOST')}:{os.getenv('PGPORT', '5432')}" + f"/{os.getenv('PGDATABASE')}" +) + +engine = create_engine(DATABASE_URL) +``` + +--- + +## Streamlit with Lakebase + +```python +import streamlit as st +import psycopg2 + +@st.cache_resource +def get_db_connection(): + return psycopg2.connect( + host=os.getenv("PGHOST"), + database=os.getenv("PGDATABASE"), + user=os.getenv("PGUSER"), + password=os.getenv("PGPASSWORD"), + ) +``` + +--- + +## Critical: requirements.txt + +`psycopg2` and `asyncpg` are **NOT pre-installed** in the Databricks Apps runtime. You **MUST** include them in `requirements.txt` or the app will crash on startup: + +``` +psycopg2-binary +``` + +For async apps: +``` +asyncpg +``` + +**This is the most common cause of Lakebase app failures.** + +## Notes + +- Lakebase is in **Public Preview** +- Each app gets its own PostgreSQL role with `Can connect and create` permission +- Lakebase is ideal alongside SQL warehouse: use Lakebase for app state, SQL warehouse for analytics diff --git a/.claude/skills/databricks-app-python/6-mcp-approach.md b/.claude/skills/databricks-app-python/6-mcp-approach.md new file mode 100644 index 00000000..23ffb67d --- /dev/null +++ b/.claude/skills/databricks-app-python/6-mcp-approach.md @@ -0,0 +1,94 @@ +# MCP Tools for App Lifecycle + +Use MCP tools to create, deploy, and manage Databricks Apps programmatically. This mirrors the CLI workflow but can be invoked by AI agents. + +--- + +## Workflow + +### Step 1: Write App Files Locally + +Create your app files in a local folder: + +``` +my_app/ +├── app.py # Main application +├── models.py # Pydantic models +├── backend.py # Data access layer +├── requirements.txt # Additional dependencies +└── app.yaml # Databricks Apps configuration +``` + +### Step 2: Upload to Workspace + +```python +# MCP Tool: upload_folder +upload_folder( + local_folder="/path/to/my_app", + workspace_folder="/Workspace/Users/user@example.com/my_app" +) +``` + +### Step 3: Create App + +```python +# MCP Tool: create_app +result = create_app( + name="my-dashboard", + description="Customer analytics dashboard" +) +# Returns: {"name": "my-dashboard", "url": "https://..."} +``` + +### Step 4: Deploy + +```python +# MCP Tool: deploy_app +result = deploy_app( + app_name="my-dashboard", + source_code_path="/Workspace/Users/user@example.com/my_app" +) +# Returns: {"deployment_id": "...", "status": "PENDING", ...} +``` + +### Step 5: Verify + +```python +# MCP Tool: get_app +app = get_app(name="my-dashboard") +# Returns: {"name": "...", "url": "...", "status": "RUNNING", ...} + +# MCP Tool: get_app_logs +logs = get_app_logs(app_name="my-dashboard") +# Returns: {"logs": "...", ...} +``` + +### Step 6: Iterate + +1. Fix issues in local files +2. Re-upload with `upload_folder` +3. Re-deploy with `deploy_app` +4. Check `get_app_logs` for errors +5. Repeat until app is healthy + +--- + +## Quick Reference: MCP Tools + +| Tool | Description | +|------|-------------| +| **`create_app`** | Create a new Databricks App | +| **`get_app`** | Get app details and status | +| **`list_apps`** | List all apps in the workspace | +| **`deploy_app`** | Deploy app from workspace source path | +| **`delete_app`** | Delete an app | +| **`get_app_logs`** | Get app deployment and runtime logs | +| **`upload_folder`** | Upload local folder to workspace (shared tool) | + +--- + +## Notes + +- Add resources (SQL warehouse, Lakebase, etc.) via the Databricks Apps UI after creating the app +- MCP tools use the service principal's permissions — ensure it has access to required resources +- For manual deployment, see [4-deployment.md](4-deployment.md) diff --git a/.claude/skills/databricks-app-python/README.md b/.claude/skills/databricks-app-python/README.md deleted file mode 100644 index 63b30a6b..00000000 --- a/.claude/skills/databricks-app-python/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Databricks Python App Skill - -Claude Agent skill for building Python-based Databricks applications with various frameworks. - -## Structure - -``` -databricks-app-python/ -├── SKILL.md # Main skill file with core patterns -├── dash.md # Dash framework specific guide -├── streamlit.md # Streamlit guide (coming soon) -├── flask.md # Flask guide (coming soon) -└── README.md # This file -``` - -## Overview - -This skill provides comprehensive guidance for building Python applications for Databricks, including: - -### Core Components (SKILL.md) -- Architecture patterns -- Pydantic data models -- Mock and real backend patterns -- Databricks connectivity -- Unity Catalog integration -- Environment configuration -- Best practices - -### Framework-Specific Guides - -#### Dash (dash.md) -- Complete Dash application structure -- Component patterns (cards, tables, charts, modals) -- Callback patterns and best practices -- Plotly chart examples -- Bootstrap styling -- Common pitfalls and solutions - -#### Coming Soon -- **streamlit.md** - Streamlit patterns for rapid prototyping -- **flask.md** - Flask patterns for custom web apps -- **gradio.md** - Gradio for ML model interfaces - -## Usage - -When user requests a Python app for Databricks: - -1. **SKILL.md** provides the foundation: - - Data model design - - Backend architecture (mock + real) - - Databricks connectivity patterns - - Database setup - -2. **Framework-specific file** provides implementation: - - UI component patterns - - Framework-specific callbacks/routing - - Styling and theming - - Deployment configuration - -## Design Philosophy - -### Separation of Concerns -- **Core patterns** (SKILL.md) - Framework-agnostic -- **Framework details** (dash.md, etc.) - Implementation specifics - -### Progressive Complexity -- Start with mock backend (rapid development) -- Add real backend (production ready) -- Scale with Unity Catalog - -### Consistent Architecture -All apps follow same pattern: -``` -models.py → Data definitions -backend_mock.py → Sample data -backend_real.py → Databricks SQL -{framework}_app.py → UI implementation -setup_database.py → Schema initialization -``` - -## Example Applications - -### Order Management (Dash) -Location: `/example-app-dash/` - -Features: -- Dashboard with statistics and charts -- Filterable orders table -- Customer and product management -- Order details modal -- Mock and real backend support - -To run: -```bash -cd example-app-dash -uv pip install -r requirements.txt -USE_MOCK_BACKEND=true uv run python dash_app.py -``` - -## Adding New Frameworks - -To add support for a new framework: - -1. Create `{framework}.md` in this directory -2. Follow the structure of `dash.md`: - - When to use - - Dependencies - - Project structure - - Component patterns - - Best practices - - Common pitfalls - - Example code - -3. Update `SKILL.md` to reference new framework -4. Create example app in `/example-app-{framework}/` - -## Contributing - -When updating this skill: - -1. **SKILL.md changes**: Update if affecting all frameworks - - New backend patterns - - Database connectivity - - Environment configuration - - Pydantic model patterns - -2. **Framework-specific changes**: Update individual files - - New component patterns - - Framework version updates - - Best practices - - Bug fixes - -3. Keep example apps in sync with documentation - -## Testing - -Before committing changes: - -1. Test example apps run without errors -2. Verify all code examples are syntactically correct -3. Check cross-references between files work -4. Ensure new patterns follow existing conventions - -## Related Skills - -- **databricks-app-apx** - APX framework (FastAPI + React) -- **databricks-dev** - General Databricks development -- **python-dev** - Python development standards -- **asset-bundles** - Databricks Asset Bundles - -## Support - -For issues or questions: -1. Check framework-specific documentation -2. Review example applications -3. Consult Databricks documentation -4. Check framework-specific communities diff --git a/.claude/skills/databricks-app-python/SKILL.md b/.claude/skills/databricks-app-python/SKILL.md index 1bbdeaeb..eb62551b 100644 --- a/.claude/skills/databricks-app-python/SKILL.md +++ b/.claude/skills/databricks-app-python/SKILL.md @@ -1,812 +1,208 @@ --- name: databricks-app-python -description: "Build Python-based Databricks applications using Dash, Streamlit, or Flask." +description: "Builds Python-based Databricks applications using Dash, Streamlit, Gradio, Flask, FastAPI, or Reflex. Handles OAuth authorization (app and user auth), app resources, SQL warehouse and Lakebase connectivity, model serving integration, and deployment. Use when building Python web apps, dashboards, ML demos, or REST APIs for Databricks, or when the user mentions Streamlit, Dash, Gradio, Flask, FastAPI, Reflex, or Databricks app." --- # Databricks Python Application -Build Python-based Databricks applications using frameworks like Dash, Streamlit, Flask, or other Python web frameworks. +Build Python-based Databricks applications. For full examples and recipes, see the **[Databricks Apps Cookbook](https://apps-cookbook.dev/)**. -## Trigger Conditions - -**Invoke when user requests**: -- "Dash app" or "Dash application" -- "Streamlit app" or "Streamlit application" -- "Python web app" for Databricks -- Building data visualization or dashboard apps -- Order management, analytics dashboard, etc. - -**Do NOT invoke if user specifies**: APX, React, Node.js, or other non-Python frameworks. +--- -## Framework Selection +## Critical Rules (always follow) -Ask user which framework to use if not specified: -- **Dash** - Rich interactive dashboards, Bootstrap components, Plotly charts -- **Streamlit** - Rapid prototyping, simple syntax, data science focus, automatic reactivity -- **Flask** - Lightweight, flexible, custom web apps (coming soon) - -### Dash vs Streamlit Comparison - -| Aspect | Dash | Streamlit | -|--------|------|-----------| -| **Development Speed** | Moderate (more boilerplate) | Fast (script-based) | -| **Learning Curve** | Steeper (callbacks, components) | Gentle (Pythonic, intuitive) | -| **Layout Control** | High (Bootstrap grid, custom CSS) | Medium (columns, containers) | -| **Styling** | Extensive (Bootstrap themes, CSS) | Limited (custom CSS via markdown) | -| **Callbacks** | Explicit (Input/Output decorators) | Automatic (reruns on interaction) | -| **State Management** | Manual (via callbacks) | Built-in (st.session_state) | -| **Performance** | Better for complex interactions | Slower (full page reruns) | -| **Best For** | Production dashboards, BI tools | Prototypes, data science demos | -| **Multi-page Apps** | Better routing support | Simpler but less flexible | -| **Data Science Fit** | Good (requires more setup) | Excellent (notebook-like) | -| **Code Complexity** | ~600 lines for full app | ~400 lines for full app | - -**Choose Dash when:** -- Building production-grade business intelligence dashboards -- Need precise control over layout and styling -- Require complex callback chains and interactions -- Want Bootstrap components and themes -- Building for non-technical business users - -**Choose Streamlit when:** -- Rapid prototyping and POCs -- Data science team building internal tools -- Simple data exploration and visualization -- ML model demos and experiments -- Prefer notebook-like development workflow - -For framework-specific details, see: -- **[dash.md](dash.md)** - Complete Dash implementation guide -- **[streamlit.md](streamlit.md)** - Complete Streamlit implementation guide -- **flask.md** - Flask patterns (coming soon) - -## Prerequisites Check - -1. Verify Python environment: `python --version` (3.9+) -2. Check for `uv` package manager: `uv --version` -3. Verify Databricks connectivity (if using real backend): - - `DATABRICKS_WAREHOUSE_ID` (required for SQL backend) - - Databricks CLI configured profile (SDK Config handles auth automatically) - - **Note:** No explicit tokens needed when using SDK Config approach +- **MUST** confirm framework choice or use [Framework Selection](#framework-selection) below +- **MUST** use SDK `Config()` for authentication (never hardcode tokens) +- **MUST** use `app.yaml` `valueFrom` for resources (never hardcode resource IDs) +- **MUST** use `dash-bootstrap-components` for Dash app layout and styling +- **MUST** use `@st.cache_resource` for Streamlit database connections +- **MUST** deploy Flask with Gunicorn, FastAPI with uvicorn (not dev servers) -## Core Architecture - -All Python Databricks apps follow this pattern: +## Required Steps +Copy this checklist and verify each item: ``` -app-directory/ -├── models.py # Pydantic data models -├── backend_mock.py # Mock backend with sample data -├── backend_real.py # Real Databricks backend -├── {framework}_app.py # Main application (dash_app.py, streamlit_app.py, etc.) -├── setup_database.py # Database initialization -├── requirements.txt # Python dependencies -├── app.yaml # Databricks Apps configuration -├── .env # Environment configuration -└── README.md # Documentation +- [ ] Framework selected +- [ ] Auth strategy decided: app auth, user auth, or both +- [ ] App resources identified (SQL warehouse, Lakebase, serving endpoint, etc.) +- [ ] Backend data strategy decided (SQL warehouse, Lakebase, or SDK) +- [ ] Deployment method: CLI or DABs ``` -### Framework-Specific Requirements - -**Dash (dash_app.py):** -```txt -dash>=2.14.0 -dash-bootstrap-components>=1.5.0 -pandas>=2.0.0 -plotly>=5.17.0 -pydantic>=2.0.0 -python-dotenv>=1.0.0 -databricks-sdk>=0.12.0 -databricks-sql-connector>=3.0.0 -``` +--- -**Streamlit (streamlit_app.py):** -```txt -streamlit>=1.28.0 -pandas>=2.0.0 -plotly>=5.17.0 -pydantic>=2.0.0 -python-dotenv>=1.0.0 -databricks-sdk>=0.12.0 -databricks-sql-connector>=3.0.0 -``` +## Framework Selection -**Key Difference:** Dash requires `dash-bootstrap-components`, Streamlit doesn't need any additional UI libraries. +| Framework | Best For | app.yaml Command | +|-----------|----------|------------------| +| **Dash** | Production dashboards, BI tools, complex interactivity | `["python", "app.py"]` | +| **Streamlit** | Rapid prototyping, data science apps, internal tools | `["streamlit", "run", "app.py"]` | +| **Gradio** | ML demos, model interfaces, chat UIs | `["python", "app.py"]` | +| **Flask** | Custom REST APIs, lightweight apps, webhooks | `["gunicorn", "app:app", "-w", "4", "-b", "0.0.0.0:8080"]` | +| **FastAPI** | Async APIs, auto-generated OpenAPI docs | `["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]` | +| **Reflex** | Full-stack Python apps without JavaScript | `["reflex", "run", "--env", "prod"]` | -## Workflow Overview +**Default**: Recommend **Streamlit** for prototypes, **Dash** for production dashboards, **FastAPI** for APIs, **Gradio** for ML demos. -### Phase 1: Planning & Models (10-15 min) -1. Understand requirements -2. Design data models -3. Create Pydantic models with validation -4. Create TodoWrite to track progress +--- -### Phase 2: Mock Backend (10-15 min) -1. Generate realistic sample data -2. Implement filtering and search -3. Create statistics methods -4. Test data generation +## Quick Reference -### Phase 3: Application UI (20-30 min) -1. Set up framework structure -2. Create consistent styling -3. Build main pages/views -4. Add interactivity (filters, charts) -5. Implement data tables +| Concept | Details | +|---------|---------| +| **Runtime** | Python 3.11, Ubuntu 22.04, 2 vCPU, 6 GB RAM | +| **Pre-installed** | Dash 2.18.1, Streamlit 1.38.0, Gradio 4.44.0, Flask 3.0.3, FastAPI 0.115.0 | +| **Auth (app)** | Service principal via `Config()` — auto-injected `DATABRICKS_CLIENT_ID`/`DATABRICKS_CLIENT_SECRET` | +| **Auth (user)** | `x-forwarded-access-token` header — see [1-authorization.md](1-authorization.md) | +| **Resources** | `valueFrom` in app.yaml — see [2-app-resources.md](2-app-resources.md) | +| **Cookbook** | https://apps-cookbook.dev/ | +| **Docs** | https://docs.databricks.com/aws/en/dev-tools/databricks-apps/ | -### Phase 4: Real Backend (15-20 min) -1. Design Unity Catalog schema -2. Implement SQL queries -3. Create database initialization -4. Add data seeding from mock +--- -### Phase 5: Testing & Documentation (10-15 min) -1. Test with mock backend -2. Test with real backend -3. Create comprehensive README -4. Add deployment instructions +## Detailed Guides -## Databricks Connectivity Patterns +**Authorization**: Use [1-authorization.md](1-authorization.md) when configuring app or user authorization — covers service principal auth, on-behalf-of user tokens, OAuth scopes, and per-framework code examples. (Keywords: OAuth, service principal, user auth, on-behalf-of, access token, scopes) -### Environment Configuration +**App resources**: Use [2-app-resources.md](2-app-resources.md) when connecting your app to Databricks resources — covers SQL warehouses, Lakebase, model serving, secrets, volumes, and the `valueFrom` pattern. (Keywords: resources, valueFrom, SQL warehouse, model serving, secrets, volumes, connections) -```python -# Standard environment variables -USE_MOCK_BACKEND=true|false # Toggle backend mode -DATABRICKS_WAREHOUSE_ID=... # SQL Warehouse ID (required) -DATABRICKS_CATALOG=main # Unity Catalog -DATABRICKS_SCHEMA=app_schema # Schema name -DATABRICKS_APP_PORT=8080 # Application port -DEBUG=false # Debug mode - -# Note: No DATABRICKS_TOKEN needed when using SDK Config -# Authentication handled automatically via: -# - Databricks CLI profile (local development) -# - Service principal (Databricks Apps) -``` +**Frameworks**: See [3-frameworks.md](3-frameworks.md) for Databricks-specific patterns per framework — covers Dash, Streamlit, Gradio, Flask, FastAPI, and Reflex with auth integration, deployment commands, and Cookbook links. (Keywords: Dash, Streamlit, Gradio, Flask, FastAPI, Reflex, framework selection) -### Backend Toggle Pattern +**Deployment**: Use [4-deployment.md](4-deployment.md) when deploying your app — covers Databricks CLI, Asset Bundles (DABs), app.yaml configuration, and post-deployment verification. (Keywords: deploy, CLI, DABs, asset bundles, app.yaml, logs) -```python -import os +**Lakebase**: Use [5-lakebase.md](5-lakebase.md) when using Lakebase (PostgreSQL) as your app's data layer — covers auto-injected env vars, psycopg2/asyncpg patterns, and when to choose Lakebase vs SQL warehouse. (Keywords: Lakebase, PostgreSQL, psycopg2, asyncpg, transactional, PGHOST) -USE_MOCK = os.getenv("USE_MOCK_BACKEND", "true").lower() == "true" +**MCP tools**: Use [6-mcp-approach.md](6-mcp-approach.md) for managing app lifecycle via MCP tools — covers creating, deploying, monitoring, and deleting apps programmatically. (Keywords: MCP, create app, deploy app, app logs) -if USE_MOCK: - from backend_mock import MockBackend - backend = MockBackend() -else: - from backend_real import RealBackend - backend = RealBackend() -``` +--- -### Pydantic Models Pattern +## Workflow -```python -from pydantic import BaseModel, Field, field_validator -from decimal import Decimal -from datetime import datetime -from enum import Enum -from typing import List, Optional +1. Determine the task type: -class StatusEnum(str, Enum): - """Status enumeration""" - ACTIVE = "active" - INACTIVE = "inactive" - -class Entity(BaseModel): - """Main entity model""" - id: str = Field(..., description="Unique identifier") - name: str = Field(..., description="Entity name") - created_at: datetime = Field(default_factory=datetime.utcnow) - status: StatusEnum = Field(default=StatusEnum.ACTIVE) - amount: Decimal = Field(..., description="Monetary amount", gt=0) - - @field_validator('amount', mode='before') - @classmethod - def validate_amount(cls, v): - """Ensure amount is a valid Decimal""" - if isinstance(v, (int, float, str)): - return Decimal(str(v)) - return v - - class Config: - json_schema_extra = { - "example": { - "id": "ENT-001", - "name": "Example Entity", - "status": "active", - "amount": "99.99" - } - } -``` + **New app from scratch?** → Use [Framework Selection](#framework-selection), then read [3-frameworks.md](3-frameworks.md) + **Setting up authorization?** → Read [1-authorization.md](1-authorization.md) + **Connecting to data/resources?** → Read [2-app-resources.md](2-app-resources.md) + **Using Lakebase (PostgreSQL)?** → Read [5-lakebase.md](5-lakebase.md) + **Deploying to Databricks?** → Read [4-deployment.md](4-deployment.md) + **Using MCP tools?** → Read [6-mcp-approach.md](6-mcp-approach.md) -### Mock Backend Pattern +2. Follow the instructions in the relevant guide +3. For full code examples, browse https://apps-cookbook.dev/ -```python -from typing import List, Optional -from models import Entity - -class MockBackend: - """Mock backend with sample data""" - - def __init__(self): - self.entities = self._generate_entities() - - def _generate_entities(self) -> List[Entity]: - """Generate sample data""" - return [ - Entity(id="ENT-001", name="Entity 1", amount=Decimal("100.00")), - Entity(id="ENT-002", name="Entity 2", amount=Decimal("200.00")), - ] - - def get_entities(self, filter_criteria: Optional[dict] = None) -> List[Entity]: - """Get entities with optional filtering""" - results = self.entities - - if filter_criteria: - # Apply filters - if filter_criteria.get("status"): - results = [e for e in results if e.status == filter_criteria["status"]] - - return results - - def get_entity(self, entity_id: str) -> Optional[Entity]: - """Get specific entity""" - for entity in self.entities: - if entity.id == entity_id: - return entity - return None - - def get_statistics(self) -> dict: - """Get aggregated statistics""" - return { - "total_count": len(self.entities), - "total_amount": float(sum(e.amount for e in self.entities)) - } -``` +--- -### Real Backend Pattern (Databricks SQL) +## Core Architecture -**Important:** For SQL Warehouse connection examples, see the Databricks Apps Cookbook: -- **Tables Read Example**: https://apps-cookbook.dev/docs/dash/tables/tables_read -- Shows proper service principal authentication using SDK Config +All Python Databricks apps follow this pattern: -```python -import os -from databricks import sql -from databricks.sdk import WorkspaceClient -from databricks.sdk.core import Config -from typing import List, Optional -from models import Entity - -class RealBackend: - """Real backend using Databricks SQL with SDK Config authentication""" - - def __init__(self, catalog: Optional[str] = None, schema: Optional[str] = None): - self.catalog = catalog or os.getenv("DATABRICKS_CATALOG", "main") - self.schema = schema or os.getenv("DATABRICKS_SCHEMA", "app_schema") - self.warehouse_id = os.getenv("DATABRICKS_WAREHOUSE_ID") - - if not self.warehouse_id: - raise ValueError("DATABRICKS_WAREHOUSE_ID required") - - self.config = Config() # Automatically handles authentication - self._connection = None - - def _get_connection(self): - """Get or create database connection using SDK Config""" - if self._connection is None: - self._connection = sql.connect( - server_hostname=self.config.host, - http_path=f"/sql/1.0/warehouses/{self.warehouse_id}", - credentials_provider=lambda: self.config.authenticate - ) - return self._connection - - def _execute_query(self, query: str, params: Optional[dict] = None) -> List[dict]: - """Execute SQL query and return results""" - connection = self._get_connection() - cursor = connection.cursor() - - try: - cursor.execute(query, params or {}) - columns = [desc[0] for desc in cursor.description] - results = [] - for row in cursor.fetchall(): - results.append(dict(zip(columns, row))) - return results - finally: - cursor.close() - - def get_entities(self, filter_criteria: Optional[dict] = None) -> List[Entity]: - """Get entities with optional filtering""" - query = f""" - SELECT * FROM {self.catalog}.{self.schema}.entities - WHERE 1=1 - """ - - params = {} - if filter_criteria and filter_criteria.get("status"): - query += " AND status = :status" - params["status"] = filter_criteria["status"] - - query += " ORDER BY created_at DESC" - - results = self._execute_query(query, params) - return [Entity(**row) for row in results] - - def initialize_schema(self): - """Initialize database schema""" - self._execute_query(f""" - CREATE TABLE IF NOT EXISTS {self.catalog}.{self.schema}.entities ( - id STRING NOT NULL, - name STRING NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), - status STRING NOT NULL, - amount DECIMAL(10, 2) NOT NULL, - PRIMARY KEY (id) - ) - """) - - def close(self): - """Close database connection""" - if self._connection: - self._connection.close() - self._connection = None +``` +app-directory/ +├── app.py # Main application (or framework-specific name) +├── models.py # Pydantic data models +├── backend.py # Data access layer +├── requirements.txt # Additional Python dependencies +├── app.yaml # Databricks Apps configuration +└── README.md ``` -### Database Setup Script Pattern +### Backend Toggle Pattern ```python -"""Database setup script""" import os -import argparse -from dotenv import load_dotenv -from backend_mock import MockBackend -from backend_real import RealBackend - -def setup_database(seed_data: bool = False): - """Initialize database and optionally seed data""" - load_dotenv() - - # Verify environment - required_vars = ["DATABRICKS_SERVER_HOSTNAME", "DATABRICKS_TOKEN", "DATABRICKS_WAREHOUSE_ID"] - missing = [v for v in required_vars if not os.getenv(v)] - if missing: - print(f"Missing: {', '.join(missing)}") - return 1 - - # Initialize backend - backend = RealBackend() - backend.initialize_schema() - - # Seed if requested - if seed_data: - mock = MockBackend() - # Copy data from mock to real backend - for entity in mock.entities: - backend.insert_entity(entity) - - backend.close() - return 0 - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--seed", action="store_true") - args = parser.parse_args() - exit(setup_database(seed_data=args.seed)) -``` - -## Best Practices - -### Data Models -- Use Pydantic for validation -- Include proper type hints -- Add `json_schema_extra` examples -- Handle Decimal for currency -- Use Enums for status fields - -### Backend Design -- Create both mock and real backends -- Use consistent interface between them -- Implement filtering and pagination -- Provide statistics/aggregations -- Use parameterized queries (security) - -### Error Handling -- Validate environment variables -- Handle connection failures gracefully -- Provide clear error messages -- Log errors appropriately - -### Configuration -- Use `.env` files for configuration -- Never commit secrets -- Provide `.env.example` template -- Support environment variable overrides - -### Testing Strategy -1. Start with mock backend (rapid development) -2. Test all features with sample data -3. Initialize real database with `--seed` -4. Test with real backend -5. Verify performance at scale - -## Common Patterns - -### Decimal Handling -```python -# Always convert to Decimal for monetary values -@field_validator('price', 'total', mode='before') -@classmethod -def validate_decimal(cls, v): - if isinstance(v, (int, float, str)): - return Decimal(str(v)) - return v -``` +from databricks.sdk.core import Config -### Date Formatting -```python -# Consistent date formatting -order.order_date.strftime("%Y-%m-%d %H:%M") -order.created_at.isoformat() -``` +USE_MOCK = os.getenv("USE_MOCK_BACKEND", "true").lower() == "true" -### Status Colors -```python -# Map status to visual indicators -STATUS_COLORS = { - Status.ACTIVE: "#2CA02C", # Green - Status.PENDING: "#FF7F0E", # Orange - Status.FAILED: "#D62728", # Red -} -``` +if USE_MOCK: + from backend_mock import MockBackend as Backend +else: + from backend_real import RealBackend as Backend -### Filtering Pattern -```python -# Reusable filter criteria model -class FilterCriteria(BaseModel): - status: Optional[Status] = None - date_from: Optional[datetime] = None - date_to: Optional[datetime] = None - search: Optional[str] = None +backend = Backend() ``` -## Success Criteria - -- [ ] Pydantic models with proper validation -- [ ] Mock backend with realistic data -- [ ] Framework UI with consistent styling -- [ ] Real backend with Unity Catalog -- [ ] Database initialization script -- [ ] Environment configuration -- [ ] Comprehensive documentation -- [ ] Both backends tested and working - -## Troubleshooting +### SQL Warehouse Connection (shared across all frameworks) -**First Step: Check Application Logs** -```bash -# Always check logs first when troubleshooting -databricks apps logs --profile - -# Examples: -databricks apps logs order-management-dash-dev -p DEFAULT -databricks apps logs order-management-streamlit-dev -p DEFAULT -``` - -Logs reveal: -- Deployment errors and stack traces -- Backend connection status (look for "✅ Initialized real backend") -- Missing dependencies or import errors -- SQL connection failures -- App startup issues - -**Connection Issues** -- Verify Databricks CLI profile is configured: `databricks auth profiles` -- Check `DATABRICKS_WAREHOUSE_ID` exists and is accessible -- Ensure warehouse is running: `databricks warehouses get ` -- Verify network connectivity to workspace -- For service principal: Check permissions on warehouse and catalog -- **Check logs for connection errors:** `databricks apps logs ` - -**Data Type Errors** -- Use Decimal for monetary values -- Handle None/Optional properly -- Validate datetime parsing - -**Performance Issues** -- Add database indexes -- Implement pagination -- Use query result caching -- Optimize SQL queries - -## Deployment to Databricks - -### Ask User for Deployment Preference - -**IMPORTANT:** Before deploying, ask the user which deployment method they prefer: - -1. **Databricks CLI** - Simple, direct deployment using `databricks apps` commands -2. **Databricks Asset Bundles (DABs)** - Infrastructure-as-code approach with version control - -Example: "Would you like to deploy using Databricks CLI or Databricks Asset Bundles (DABs)?" - -### Option 1: Deploy with Databricks CLI - -**Prerequisites:** -- Databricks CLI installed -- Authenticated profile configured -- SQL Warehouse ID available - -**Steps:** - -1. **Create app.yaml** - -**For Dash apps:** -```yaml -command: - - "python" - - "dash_app.py" - -env: - - name: USE_MOCK_BACKEND - value: "false" - - name: DATABRICKS_WAREHOUSE_ID - value: "your-warehouse-id" - - name: DATABRICKS_CATALOG - value: "main" - - name: DATABRICKS_SCHEMA - value: "app_schema" - - name: DATABRICKS_APP_PORT - value: "8080" - - name: DEBUG - value: "false" -``` +```python +from databricks.sdk.core import Config +from databricks import sql -**For Streamlit apps:** -```yaml -command: - - "streamlit" - - "run" - - "streamlit_app.py" - - "--server.port" - - "8080" - - "--server.address" - - "0.0.0.0" - -env: - - name: USE_MOCK_BACKEND - value: "false" - - name: DATABRICKS_WAREHOUSE_ID - value: "your-warehouse-id" - - name: DATABRICKS_CATALOG - value: "main" - - name: DATABRICKS_SCHEMA - value: "app_schema" +cfg = Config() # Auto-detects credentials from environment +conn = sql.connect( + server_hostname=cfg.host, + http_path=f"/sql/1.0/warehouses/{os.getenv('DATABRICKS_WAREHOUSE_ID')}", + credentials_provider=lambda: cfg.authenticate, +) ``` -**Note:** Streamlit uses `streamlit run` command, while Dash uses `python`. Streamlit doesn't need `DATABRICKS_APP_PORT` env var as it's specified in the command. +### Pydantic Models -2. **Initialize database schema** -```bash -# Run setup script locally (requires profile configured) -python setup_database.py --seed -``` - -3. **Create Databricks app** -```bash -databricks apps create --profile -``` +```python +from pydantic import BaseModel, Field +from datetime import datetime +from enum import Enum -4. **Upload source code to workspace** -```bash -databricks workspace mkdirs /Workspace/Users//apps/ --profile -databricks workspace import-dir . /Workspace/Users//apps/ --profile -``` +class Status(str, Enum): + ACTIVE = "active" + PENDING = "pending" -5. **Deploy the app** -```bash -databricks apps deploy \ - --source-code-path /Workspace/Users//apps/ \ - --profile -``` +class EntityOut(BaseModel): + id: str + name: str + status: Status + created_at: datetime -6. **Get app URL** -```bash -databricks apps get --profile +class EntityIn(BaseModel): + name: str = Field(..., min_length=1) + status: Status = Status.PENDING ``` -**Redeployment:** -```bash -# Update workspace files -databricks workspace delete /Workspace/Users//apps/ --recursive --profile -databricks workspace mkdirs /Workspace/Users//apps/ --profile -databricks workspace import-dir . /Workspace/Users//apps/ --profile - -# Redeploy -databricks apps deploy \ - --source-code-path /Workspace/Users//apps/ \ - --profile -``` +--- -### Option 2: Deploy with Databricks Asset Bundles (DABs) - -**Prerequisites:** -- Databricks CLI installed (v0.239.0+) -- App already deployed via CLI (recommended workflow) - -**Advantages:** -- Version controlled deployment -- Multi-environment support (dev/staging/prod) -- Declarative infrastructure -- Easier CI/CD integration - -**Recommended Workflow: CLI First, Then DABs** - -1. **Deploy app using CLI first** (see Option 1 above) - - This creates the app and validates everything works - - Easier to debug issues initially - -2. **Generate bundle configuration from existing app** -```bash -# This creates resources/*.app.yml and downloads source to src/app/ -databricks bundle generate app \ - --existing-app-name \ - --key \ - --profile - -# Example: -databricks bundle generate app \ - --existing-app-name order-management-dash \ - --key order_management_dash \ - --profile DEFAULT -``` +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Connection exhausted** | Use `@st.cache_resource` (Streamlit) or connection pooling | +| **Auth token not found** | Check `x-forwarded-access-token` header — only available when deployed, not locally | +| **App won't start** | Check `app.yaml` command matches framework; check `databricks apps logs ` | +| **Resource not accessible** | Add resource via UI, verify SP has permissions, use `valueFrom` in app.yaml | +| **Import error on deploy** | Add missing packages to `requirements.txt` (pre-installed packages don't need listing) | +| **Lakebase app crashes on start** | `psycopg2`/`asyncpg` are NOT pre-installed — MUST add to `requirements.txt` | +| **Port conflict** | Databricks Apps expects port 8080; configure your framework accordingly | +| **Streamlit: set_page_config error** | `st.set_page_config()` must be the first Streamlit command | +| **Dash: unstyled layout** | Add `dash-bootstrap-components`; use `dbc.themes.BOOTSTRAP` | +| **Slow queries** | Use Lakebase for transactional/low-latency; SQL warehouse for analytical queries | -**What gets generated:** -- `resources/.app.yml` - Minimal app resource definition -- `src/app/` - All app source files including `app.yaml` with env vars -- `databricks.yml` updated with bundle structure - -3. **Update generated configuration for multi-environment** - -**Edit `databricks.yml`:** -```yaml -bundle: - name: - -include: - - resources/*.yml - -variables: - warehouse_id: - default: "your-warehouse-id" - catalog: - default: "main" - schema: - default: "app_schema" - -targets: - dev: - default: true - mode: development - workspace: - profile: - variables: - warehouse_id: "dev-warehouse-id" - schema: "app_schema_dev" - - prod: - mode: production - workspace: - profile: - variables: - warehouse_id: "prod-warehouse-id" - schema: "app_schema_prod" -``` +--- -**Edit `resources/.app.yml`:** -```yaml -resources: - apps: - : - name: -${bundle.target} # Environment-specific naming - description: "Python ${framework} application" - source_code_path: ../src/app # Or .. if source in project root -``` +## Platform Constraints -**Important:** Environment variables are in `src/app/app.yaml`, NOT in databricks.yml: -```yaml -command: - - "python" - - "dash_app.py" - -env: - - name: USE_MOCK_BACKEND - value: "false" - - name: DATABRICKS_WAREHOUSE_ID - value: "your-warehouse-id" - - name: DATABRICKS_CATALOG - value: "main" - - name: DATABRICKS_SCHEMA - value: "app_schema" -``` +| Constraint | Details | +|------------|---------| +| **Runtime** | Python 3.11, Ubuntu 22.04 LTS | +| **Compute** | 2 vCPUs, 6 GB memory (default) | +| **Pre-installed frameworks** | Dash, Streamlit, Gradio, Flask, FastAPI, Shiny | +| **Custom packages** | Add to `requirements.txt` in app root | +| **Network** | Apps can reach Databricks APIs; external access depends on workspace config | +| **User auth** | Public Preview — workspace admin must enable before adding scopes | -4. **Deploy and run** -```bash -# Validate configuration -databricks bundle validate -t dev +--- -# Deploy to dev (creates/updates resource) -databricks bundle deploy -t dev +## Official Documentation -# Start the app (required after deployment) -databricks bundle run -t dev +- **[Databricks Apps Overview](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/)** — main docs hub +- **[Apps Cookbook](https://apps-cookbook.dev/)** — ready-to-use code snippets (Streamlit, Dash, Reflex, FastAPI) +- **[Authorization](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/auth)** — app auth and user auth +- **[Resources](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources)** — SQL warehouse, Lakebase, serving, secrets +- **[app.yaml Reference](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/app-runtime)** — command and env config +- **[System Environment](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/system-env)** — pre-installed packages, runtime details -# For production -databricks bundle deploy -t prod -databricks bundle run -t prod -``` +## Related Skills -**Key Differences from Other Resources:** -- Environment variables go in `app.yaml` (source dir), NOT databricks.yml -- Apps have minimal bundle configuration (name, description, path) -- Must run `databricks bundle run` to start the app after deployment - -**For complete DABs guidance, use the `asset-bundles` skill.** - -### Post-Deployment Steps - -1. **Verify deployment** - - Access app URL - - Check all pages load - - Verify data from Unity Catalog - -2. **Configure permissions** - - Set up user access - - Configure service principal permissions - - Grant warehouse access - -3. **Set up monitoring and view logs** - - **View application logs:** - ```bash - # View logs for your deployed app - databricks apps logs --profile - - # Examples: - databricks apps logs order-management-dash-dev --profile DEFAULT - databricks apps logs order-management-streamlit-dev --profile DEFAULT - ``` - - **What logs show:** - - `[SYSTEM]` - Deployment status, file updates, dependency installation - - `[APP]` - Application output (print statements, framework messages) - - Backend initialization messages - - Connection status to Unity Catalog - - Error messages and stack traces - - **Useful for debugging:** - - ✅ Verify real backend connection: Look for "✅ Initialized real backend: main.schema" - - ✅ Check dependency installation: "Requirements installed successfully" - - ✅ Confirm app start: "App started successfully" - - ✅ Diagnose connection errors: SQL connection failures - - ✅ Track deployments: Each deployment has unique ID - - **Additional monitoring:** - - Monitor warehouse usage in Databricks SQL - - Track app performance and response times - - Set up alerts for app failures - -4. **Documentation** - - Update README with deployment URL - - Document environment variables - - Add troubleshooting guide - -## Reference Materials - -For framework-specific implementation details: -- **[dash.md](dash.md)** - Complete Dash implementation guide with Bootstrap components -- **[streamlit.md](streamlit.md)** - Complete Streamlit implementation guide with caching patterns -- **flask.md** - Flask patterns (coming soon) +- **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - full-stack apps with FastAPI + React +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying apps via DABs +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - backend SDK integration +- **[databricks-lakebase-provisioned](../databricks-lakebase-provisioned/SKILL.md)** - adding persistent PostgreSQL state +- **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - serving ML models for app integration diff --git a/.claude/skills/databricks-app-python/dash.md b/.claude/skills/databricks-app-python/dash.md deleted file mode 100644 index 82e42ecc..00000000 --- a/.claude/skills/databricks-app-python/dash.md +++ /dev/null @@ -1,553 +0,0 @@ -# Dash Framework Guide - -Complete guide for building Databricks applications with Plotly Dash framework. - -## When to Use Dash - -**Best for**: -- Interactive dashboards with rich charts -- Business intelligence applications -- Data visualization heavy apps -- Multi-page applications -- Apps requiring custom styling with Bootstrap - -**Alternatives**: -- Streamlit - Simpler syntax, faster prototyping -- APX - Full-stack with React frontend - -## Dependencies - -```txt -dash>=2.14.0 -dash-bootstrap-components>=1.5.0 -plotly>=5.18.0 -pandas>=2.0.0 -databricks-sdk>=0.35.0 -databricks-sql-connector>=3.0.0 -pydantic>=2.0.0 -python-dotenv>=1.0.0 -``` - -## Project Structure - -``` -dash-app/ -├── models.py # Pydantic data models -├── backend_mock.py # Mock backend with sample data -├── backend_real.py # Databricks SQL backend -├── dash_app.py # Main Dash application -├── setup_database.py # Database initialization -├── requirements.txt # Dependencies -├── .env.example # Environment template -├── run_app.sh # Quick start script -└── README.md # Documentation -``` - -## Dash Application Structure - -### Basic Setup - -```python -import os -import dash -from dash import dcc, html, dash_table, Input, Output, State, callback -import dash_bootstrap_components as dbc -import plotly.express as px -import pandas as pd - -from backend_mock import MockBackend -from models import Status - -# Initialize backend -USE_MOCK = os.getenv("USE_MOCK_BACKEND", "true").lower() == "true" -backend = MockBackend() if USE_MOCK else RealBackend() - -# Initialize Dash app with Bootstrap theme -app = dash.Dash( - __name__, - external_stylesheets=[dbc.themes.BOOTSTRAP, dbc.icons.FONT_AWESOME], - suppress_callback_exceptions=True, - title="Application Name" -) - -# Define color scheme -COLORS = { - "primary": "#1F77B4", - "success": "#2CA02C", - "warning": "#FF7F0E", - "danger": "#D62728", - "info": "#17A2B8", - "light": "#F8F9FA", - "dark": "#343A40", -} - -# Status color mappings -STATUS_COLORS = { - Status.ACTIVE: COLORS["success"], - Status.PENDING: COLORS["warning"], - Status.FAILED: COLORS["danger"], -} - -# Bootstrap badge colors (for dbc.Badge) -STATUS_BADGE_COLORS = { - Status.ACTIVE: "success", - Status.PENDING: "warning", - Status.FAILED: "danger", -} -``` - -### Navigation Bar - -```python -def create_navbar(): - """Create navigation bar""" - return dbc.Navbar( - dbc.Container([ - dbc.Row([ - dbc.Col([ - html.I(className="fas fa-chart-line me-2"), - dbc.NavbarBrand("Application Name", className="ms-2"), - ], width="auto"), - ], align="center", className="g-0"), - dbc.Nav([ - dbc.NavItem(dbc.NavLink("Dashboard", href="/", active="exact")), - dbc.NavItem(dbc.NavLink("Orders", href="/orders", active="exact")), - dbc.NavItem(dbc.NavLink("Customers", href="/customers", active="exact")), - ], navbar=True, className="ms-auto"), - ], fluid=True), - color="dark", - dark=True, - className="mb-4" - ) -``` - -### Main Layout with Routing - -```python -app.layout = html.Div([ - dcc.Location(id='url', refresh=False), - create_navbar(), - html.Div(id='page-content', style={'minHeight': '80vh'}), - dcc.Store(id='selected-item-id'), # Client-side data storage -], style={'backgroundColor': COLORS["light"], 'minHeight': '100vh'}) - -@callback( - Output('page-content', 'children'), - Input('url', 'pathname') -) -def display_page(pathname): - """Route to different pages""" - if pathname == '/orders': - return create_orders_layout() - elif pathname == '/customers': - return create_customers_layout() - else: - return create_dashboard_layout() -``` - -## Component Patterns - -### Statistics Card - -```python -def create_stat_card(title, value, icon, color="primary", subtitle=None): - """Create a statistics card""" - return dbc.Card([ - dbc.CardBody([ - html.Div([ - html.Div([ - html.H6(title, className="text-muted mb-2"), - html.H3(value, className="mb-0"), - html.Small(subtitle, className="text-muted") if subtitle else None, - ], className="flex-grow-1"), - html.Div([ - html.I(className=f"fas {icon} fa-2x text-{color}") - ], className="ms-3"), - ], className="d-flex align-items-center"), - ]), - ], className="shadow-sm mb-3") - -# Usage -dbc.Row([ - dbc.Col(create_stat_card( - "Total Orders", - f"{stats['total_orders']:,}", - "fa-shopping-cart", - "primary" - ), md=3), - dbc.Col(create_stat_card( - "Total Revenue", - f"${stats['total_revenue']:,.2f}", - "fa-dollar-sign", - "success" - ), md=3), -]) -``` - -### Interactive Data Table - -```python -def create_data_table(data, table_id, selectable=False): - """Create interactive data table""" - if not data: - return html.Div("No data available.", className="text-muted") - - return dash_table.DataTable( - id=table_id, - data=data, - columns=[{"name": col, "id": col} for col in data[0].keys()], - page_size=20, - style_table={'overflowX': 'auto'}, - style_cell={ - 'textAlign': 'left', - 'padding': '12px', - 'fontFamily': 'Arial, sans-serif' - }, - style_header={ - 'backgroundColor': COLORS["dark"], - 'color': 'white', - 'fontWeight': 'bold' - }, - style_data_conditional=[ - { - 'if': {'row_index': 'odd'}, - 'backgroundColor': COLORS["light"] - } - ], - filter_action="native", - sort_action="native", - row_selectable='single' if selectable else False, - ) -``` - -### Plotly Charts - -```python -# Pie Chart -@callback( - Output('status-pie-chart', 'figure'), - Input('url', 'pathname') -) -def update_pie_chart(pathname): - """Create pie chart for status distribution""" - stats = backend.get_statistics() - status_data = stats['status_distribution'] - - fig = px.pie( - values=list(status_data.values()), - names=[s.title() for s in status_data.keys()], - color_discrete_sequence=px.colors.qualitative.Set3 - ) - fig.update_layout( - margin=dict(t=20, b=20, l=20, r=20), - showlegend=True, - height=300 - ) - return fig - -# Bar Chart -@callback( - Output('revenue-bar-chart', 'figure'), - Input('url', 'pathname') -) -def update_bar_chart(pathname): - """Create bar chart for revenue by category""" - data = backend.get_revenue_by_category() - - # Create color map with error handling - color_map = {} - for category in data.keys(): - color_map[category] = COLORS.get(category.lower(), COLORS["primary"]) - - fig = px.bar( - x=list(data.keys()), - y=list(data.values()), - labels={'x': 'Category', 'y': 'Revenue ($)'}, - color=list(data.keys()), - color_discrete_map=color_map - ) - fig.update_layout( - margin=dict(t=20, b=40, l=40, r=20), - showlegend=False, - height=300, - xaxis_title="", - yaxis_title="Revenue ($)" - ) - fig.update_xaxes(tickangle=-45) # Note: update_xaxes, not update_xaxis - return fig -``` - - -## Callback Patterns - -### Basic Callback - -```python -@callback( - Output('output-div', 'children'), - Input('input-button', 'n_clicks') -) -def update_output(n_clicks): - """Basic callback pattern""" - if n_clicks is None: - return "Click the button" - return f"Button clicked {n_clicks} times" -``` - -### Multiple Inputs - -```python -@callback( - Output('filtered-table', 'children'), - [Input('filter-status', 'value'), - Input('filter-date', 'value'), - Input('refresh-button', 'n_clicks')] -) -def update_table(status, date, n_clicks): - """Callback with multiple inputs""" - filter_criteria = { - "status": Status(status) if status else None, - "date": date - } - data = backend.get_data(filter_criteria) - return create_data_table(data, "result-table") -``` - -### Using State (Non-Triggering Inputs) - -```python -@callback( - Output('result', 'children'), - Input('submit-button', 'n_clicks'), - [State('input-field', 'value'), - State('dropdown', 'value')] -) -def process_form(n_clicks, input_value, dropdown_value): - """State doesn't trigger callback, only provides values""" - if n_clicks is None: - return "" - return f"Processing: {input_value}, {dropdown_value}" -``` - -### Callback Context - -```python -@callback( - Output('result', 'children'), - [Input('button1', 'n_clicks'), - Input('button2', 'n_clicks')] -) -def handle_multiple_buttons(n1, n2): - """Determine which input triggered the callback""" - ctx = dash.callback_context - - if not ctx.triggered: - return "No button clicked" - - trigger_id = ctx.triggered[0]['prop_id'].split('.')[0] - - if trigger_id == 'button1': - return "Button 1 clicked" - elif trigger_id == 'button2': - return "Button 2 clicked" - - return "Unknown trigger" -``` - -## Complete Page Example - -```python -def create_orders_layout(): - """Complete orders page with table, filters, and modal""" - return dbc.Container([ - html.H2("Orders", className="mb-4"), - - # Filters - create_filters(), - - # Data Table - dbc.Card([ - dbc.CardHeader(html.H5([ - html.I(className="fas fa-table me-2"), - "Order List" - ])), - dbc.CardBody([ - html.Div(id="orders-table") - ]), - ], className="shadow-sm mb-4"), - - # Detail Modal - create_detail_modal(), - ], fluid=True) - -@callback( - Output('orders-table', 'children'), - [Input('filter-status', 'value'), - Input('refresh-button', 'n_clicks')] -) -def update_orders_table(status, n_clicks): - """Update orders table with filters""" - filter_criteria = {"status": Status(status) if status else None} - orders = backend.get_orders(filter_criteria) - - if not orders: - return html.Div("No orders found.", className="text-muted") - - order_data = [ - { - "Order ID": o.order_id, - "Customer": o.customer_name, - "Date": o.order_date.strftime("%Y-%m-%d %H:%M"), - "Status": o.status.value.title(), - "Total": f"${float(o.total):.2f}", - } - for o in orders - ] - - return create_data_table(order_data, "orders-data-table", selectable=True) -``` - -## Best Practices - -### Performance -1. **Use `dcc.Store`** for client-side caching -2. **Implement pagination** for large datasets -3. **Use `prevent_initial_call=True`** for expensive operations -4. **Minimize callback dependencies** -5. **Cache backend queries** when appropriate - - -### Consistent Styling -```python -# Define color constants at top of file -COLORS = {...} -STATUS_COLORS = {...} -STATUS_BADGE_COLORS = {...} - -# Use consistently throughout app -dbc.Badge(status, color=STATUS_BADGE_COLORS[status]) -``` - -## Common Pitfalls - -### ❌ Wrong: Missing ID on dynamically created component -```python -def update_table(): - return dash_table.DataTable( - # Missing id! - data=data, - columns=columns - ) -``` - -### ✅ Correct: Always provide ID -```python -def update_table(): - return dash_table.DataTable( - id='dynamic-table', # Always include id - data=data, - columns=columns - ) -``` - -### ❌ Wrong: Accessing data before checking if exists -```python -@callback(...) -def toggle_modal(selected_rows, table_data, is_open): - item_id = table_data[selected_rows[0]]["id"] # May fail! -``` - -### ✅ Correct: Check before accessing -```python -@callback(..., prevent_initial_call=True) -def toggle_modal(selected_rows, table_data, is_open): - if not selected_rows or not table_data: - return False, "", "" - item_id = table_data[selected_rows[0]]["id"] # Safe -``` - -### ❌ Wrong: Using hex colors for Bootstrap badges -```python -dbc.Badge(status, color="#2CA02C") # Won't work! -``` - -### ✅ Correct: Use Bootstrap color names -```python -dbc.Badge(status, color="success") # Correct -``` - -### ❌ Wrong: Plotly method typo -```python -fig.update_xaxis(tickangle=-45) # AttributeError! -``` - -### ✅ Correct: Use plural form -```python -fig.update_xaxes(tickangle=-45) # Correct -``` - -## Running the App - -### Development Mode -```bash -# With uv -USE_MOCK_BACKEND=true DEBUG=true DATABRICKS_APP_PORT=8080 uv run python dash_app.py - -# With python directly -USE_MOCK_BACKEND=true DEBUG=true python dash_app.py -``` - -### Production Mode -```python -if __name__ == '__main__': - port = int(os.getenv("DATABRICKS_APP_PORT", "8080")) - debug = os.getenv("DEBUG", "false").lower() == "true" - - app.run(host='0.0.0.0', port=port, debug=debug) -``` - -## Deployment to Databricks - -### app.yaml -```yaml -command: - - "python" - - "dash_app.py" - -env: - - name: USE_MOCK_BACKEND - value: "false" - - name: DATABRICKS_CONFIG_PROFILE - value: "" - - name: DATABRICKS_APP_PORT - value: "8080" -``` - -### Deploy Commands -```bash -databricks apps deploy -databricks apps list -databricks apps logs -``` - - -## Additional Resources - -- **Plotly Dash Docs**: https://dash.plotly.com/ -- **Dash Bootstrap Components**: https://dash-bootstrap-components.opensource.faculty.ai/ -- **Plotly Charts**: https://plotly.com/python/ -- **Example App Snippets**: https://apps-cookbook.dev/docs/category/dash - -## Success Checklist - -- [ ] App runs with mock backend -- [ ] All pages render without errors -- [ ] Callbacks work correctly -- [ ] Filters update data tables -- [ ] Charts display properly -- [ ] Modals open and close -- [ ] Consistent styling throughout -- [ ] Empty states handled -- [ ] Error handling in callbacks -- [ ] Real backend tested -- [ ] Documentation complete diff --git a/.claude/skills/databricks-app-python/streamlit.md b/.claude/skills/databricks-app-python/streamlit.md deleted file mode 100644 index d50646e8..00000000 --- a/.claude/skills/databricks-app-python/streamlit.md +++ /dev/null @@ -1,790 +0,0 @@ -# Streamlit Framework Implementation Guide - -Complete guide for building Databricks applications with Streamlit framework. - -## Table of Contents - -- [When to Use Streamlit](#when-to-use-streamlit) -- [Core Architecture](#core-architecture) -- [Essential Patterns](#essential-patterns) -- [Best Practices](#best-practices) -- [Component Guide](#component-guide) -- [Common Pitfalls](#common-pitfalls) -- [Performance Optimization](#performance-optimization) - ---- - -## When to Use Streamlit - -**Choose Streamlit when you need:** -- Rapid prototyping and development -- Data science and ML-focused applications -- Simple, script-like development workflow -- Built-in widgets and forms -- Interactive data exploration tools -- ML model demos and POCs - -**Key Strengths:** -- ✅ Fastest development time (script-based) -- ✅ Excellent for data scientists (Pythonic) -- ✅ Built-in state management -- ✅ Automatic reactivity (reruns on interaction) -- ✅ Great for notebooks-to-apps workflow - -**Limitations:** -- ❌ Less control over layout (compared to Dash) -- ❌ Full page reruns can be slower -- ❌ Harder to build complex multi-page apps -- ❌ Limited styling customization - ---- - -## Core Architecture - -### Application Structure - -```python -""" -Streamlit App Structure -""" -import streamlit as st -from databricks.sdk.core import Config -from databricks import sql - -# 1. Page configuration (MUST be first Streamlit command) -st.set_page_config( - page_title="My App", - page_icon="📊", - layout="wide", # or "centered" - initial_sidebar_state="expanded" # or "collapsed" -) - -# 2. Backend initialization with caching -@st.cache_resource -def get_backend(): - """Initialize and cache backend connection""" - # Your backend initialization - return backend - -# 3. Initialize backend -backend = get_backend() - -# 4. Sidebar navigation -page = st.sidebar.radio("Navigation", ["Page 1", "Page 2"]) - -# 5. Page content -if page == "Page 1": - # Page 1 content - pass -elif page == "Page 2": - # Page 2 content - pass -``` - -### File Organization - -``` -streamlit-app/ -├── streamlit_app.py # Main application entry point -├── models.py # Pydantic data models -├── backend_mock.py # Mock backend with sample data -├── backend_real.py # Real Databricks backend -├── setup_database.py # Database initialization -├── requirements.txt # Python dependencies -├── app.yaml # Databricks Apps configuration -├── .env # Environment variables -└── README.md # Documentation -``` - ---- - -## Essential Patterns - -### 1. Connection Caching (Critical) - -**Always use `@st.cache_resource` for database connections:** - -```python -from databricks.sdk.core import Config -from databricks import sql - -@st.cache_resource(ttl=300, show_spinner=True) -def get_sql_connection(http_path: str): - """ - Create and cache SQL warehouse connection - - Args: - ttl: Time-to-live in seconds (5 minutes default) - show_spinner: Show loading indicator during initialization - """ - cfg = Config() # Reads DATABRICKS_HOST automatically - - return sql.connect( - server_hostname=cfg.host, - http_path=http_path, - credentials_provider=lambda: cfg.authenticate - ) - -# Usage -conn = get_sql_connection("/sql/1.0/warehouses/xxxxx") -``` - -**Why `@st.cache_resource`?** -- Persists across sessions and reruns -- Prevents connection exhaustion -- Improves performance dramatically -- Required for production apps - -### 2. Backend Toggle Pattern - -```python -import os - -@st.cache_resource -def get_backend(): - """Initialize backend based on environment""" - use_mock = os.getenv("USE_MOCK_BACKEND", "true").lower() == "true" - - if use_mock: - from backend_mock import MockBackend - return MockBackend() - else: - from backend_real import RealBackend - catalog = os.getenv("DATABRICKS_CATALOG", "main") - schema = os.getenv("DATABRICKS_SCHEMA", "app_schema") - return RealBackend(catalog=catalog, schema=schema) -``` - -### 3. Session State Management - -**Use `st.session_state` to persist data across reruns:** - -```python -# Initialize state -if 'order_id' not in st.session_state: - st.session_state.order_id = None - -# Set state -if st.button("Load Order"): - st.session_state.order_id = "ORD-001" - -# Read state -if st.session_state.order_id: - st.write(f"Current order: {st.session_state.order_id}") -``` - -**Common State Patterns:** - -```python -# Form data -if 'form_data' not in st.session_state: - st.session_state.form_data = {} - -# Page navigation -if 'current_page' not in st.session_state: - st.session_state.current_page = "Dashboard" - -# Filter persistence -if 'filters' not in st.session_state: - st.session_state.filters = { - 'status': [], - 'date_from': None, - 'date_to': None - } -``` - -### 4. Data Display Patterns - -**DataFrames (Read-only):** - -```python -import pandas as pd - -df = backend.get_orders() -st.dataframe( - df, - use_container_width=True, # Expand to container width - hide_index=True, # Hide row numbers - column_config={ - "amount": st.column_config.NumberColumn( - "Amount", - format="$%.2f" - ), - "status": st.column_config.SelectColumn( - "Status", - options=["pending", "confirmed", "shipped"] - ) - } -) -``` - -**Data Editor (Editable):** - -```python -# For editable tables -edited_df = st.data_editor( - df, - num_rows="dynamic", # Allow add/delete rows - hide_index=True, - column_config={ - "amount": st.column_config.NumberColumn( - "Amount", - min_value=0, - max_value=10000, - step=0.01, - format="$%.2f" - ) - } -) - -# Detect changes -if st.button("Save Changes"): - # Compare original vs edited - df_diff = pd.concat([df, edited_df]).drop_duplicates(keep=False) - if not df_diff.empty: - backend.update_data(edited_df) - st.success("Changes saved!") -``` - -### 5. Sidebar Navigation - -```python -# Sidebar navigation pattern -st.sidebar.title("📊 My App") -st.sidebar.markdown("---") - -page = st.sidebar.radio( - "Navigation", - ["Dashboard", "Orders", "Customers", "Products"], - label_visibility="collapsed" # Hide "Navigation" label -) - -# Filters in sidebar -st.sidebar.markdown("---") -st.sidebar.markdown("### Filters") - -status_filter = st.sidebar.multiselect( - "Status", - options=["pending", "confirmed", "shipped"], - default=None -) - -date_range = st.sidebar.date_input( - "Date Range", - value=None -) -``` - -### 6. Metrics Display - -```python -# Four-column metrics -col1, col2, col3, col4 = st.columns(4) - -with col1: - st.metric( - label="Total Orders", - value="1,234", - delta="12%", # Optional change indicator - delta_color="normal" # "normal", "inverse", or "off" - ) - -with col2: - st.metric( - label="Revenue", - value="$45,678", - delta="-8%", - delta_color="inverse" # Red for negative when inverse - ) -``` - -### 7. Charts Integration - -**Plotly Charts (Recommended):** - -```python -import plotly.express as px -import plotly.graph_objects as go - -# Pie chart -fig = px.pie( - df, - values='count', - names='status', - color='status', - color_discrete_map={'pending': '#FFA500', 'confirmed': '#2CA02C'} -) -st.plotly_chart(fig, use_container_width=True) - -# Bar chart -fig = px.bar( - df, - x='month', - y='revenue', - color='category' -) -fig.update_layout(showlegend=False) -st.plotly_chart(fig, use_container_width=True) -``` - -**Native Streamlit Charts:** - -```python -# For simple charts (less customizable but faster) -st.line_chart(df[['date', 'revenue']]) -st.bar_chart(df[['category', 'count']]) -st.area_chart(df[['date', 'cumulative_revenue']]) -``` - -### 8. Forms and Inputs - -**Form Pattern (Prevents reruns on every input change):** - -```python -with st.form("order_form"): - st.write("Create New Order") - - customer = st.text_input("Customer Name") - product = st.selectbox("Product", ["Product A", "Product B"]) - quantity = st.number_input("Quantity", min_value=1, value=1) - notes = st.text_area("Notes") - - # Form is only submitted when button is clicked - submitted = st.form_submit_button("Create Order") - - if submitted: - # Process form data - backend.create_order(customer, product, quantity, notes) - st.success("Order created!") -``` - -**Without Forms (Immediate reactivity):** - -```python -# These trigger reruns on every change -name = st.text_input("Name") -age = st.slider("Age", 0, 100) - -if st.button("Submit"): - # Only runs when button clicked - st.write(f"{name} is {age} years old") -``` - ---- - -## Best Practices - -### Page Configuration - -**✅ ALWAYS set page config first:** - -```python -# MUST be the first Streamlit command -st.set_page_config( - page_title="Order Management", - page_icon="📦", - layout="wide", - initial_sidebar_state="expanded" -) -``` - -**❌ Common mistake:** - -```python -import streamlit as st - -st.title("My App") # ❌ Error: set_page_config must be first -st.set_page_config(...) # ❌ Too late! -``` - -### State Management - -**✅ Initialize state at the top:** - -```python -# Initialize all state variables together -if 'user_id' not in st.session_state: - st.session_state.user_id = None -if 'filters' not in st.session_state: - st.session_state.filters = {} -``` - -**❌ Don't initialize in conditionals:** - -```python -# ❌ Bad: state only initialized if condition is true -if some_condition: - if 'user_id' not in st.session_state: - st.session_state.user_id = None -``` - -### Caching - -**✅ Cache expensive operations:** - -```python -@st.cache_resource # For connections, models -def get_connection(): - return create_connection() - -@st.cache_data(ttl=60) # For data, with TTL -def load_data(): - return fetch_data() -``` - -**When to use which:** -- `@st.cache_resource`: Connections, ML models, non-serializable objects -- `@st.cache_data`: DataFrames, lists, dicts, serializable data - -### Layout Organization - -**✅ Use columns for horizontal layout:** - -```python -col1, col2, col3 = st.columns([2, 1, 1]) # Ratios: 2:1:1 - -with col1: - st.write("Main content") - -with col2: - st.write("Sidebar content") -``` - -**✅ Use expanders for collapsible sections:** - -```python -with st.expander("Advanced Filters"): - filter1 = st.selectbox("Filter 1", options) - filter2 = st.multiselect("Filter 2", options) -``` - -**✅ Use tabs for switching content:** - -```python -tab1, tab2, tab3 = st.tabs(["Tab 1", "Tab 2", "Tab 3"]) - -with tab1: - st.write("Tab 1 content") - -with tab2: - st.write("Tab 2 content") -``` - -### Error Handling - -**✅ Graceful error handling:** - -```python -try: - data = backend.get_data() - st.dataframe(data) -except Exception as e: - st.error(f"Error loading data: {str(e)}") - st.info("Please check your connection and try again") -``` - -**✅ Input validation:** - -```python -user_input = st.text_input("Enter warehouse ID") - -if user_input: - if not user_input.startswith("/sql/"): - st.warning("Warehouse path should start with /sql/") - else: - # Process input - pass -``` - ---- - -## Component Guide - -### Core Components - -| Component | Use Case | Example | -|-----------|----------|---------| -| `st.title()` | Page titles | `st.title("Dashboard")` | -| `st.header()` | Section headers | `st.header("Overview")` | -| `st.subheader()` | Subsection headers | `st.subheader("Metrics")` | -| `st.text()` | Plain text | `st.text("Simple text")` | -| `st.markdown()` | Formatted text | `st.markdown("**Bold** text")` | -| `st.write()` | Auto-formatted | `st.write("Text", df, chart)` | - -### Input Widgets - -| Widget | Use Case | Example | -|--------|----------|---------| -| `st.button()` | Actions | `if st.button("Submit"):` | -| `st.text_input()` | Single-line text | `name = st.text_input("Name")` | -| `st.text_area()` | Multi-line text | `notes = st.text_area("Notes")` | -| `st.number_input()` | Numbers | `age = st.number_input("Age", 0, 100)` | -| `st.selectbox()` | Single selection | `choice = st.selectbox("Pick", options)` | -| `st.multiselect()` | Multiple selection | `choices = st.multiselect("Pick", options)` | -| `st.slider()` | Range selection | `value = st.slider("Value", 0, 100)` | -| `st.date_input()` | Date picker | `date = st.date_input("Date")` | -| `st.checkbox()` | Boolean toggle | `if st.checkbox("Agree"):` | -| `st.radio()` | Single choice | `choice = st.radio("Pick", options)` | - -### Display Components - -| Component | Use Case | Example | -|-----------|----------|---------| -| `st.dataframe()` | Read-only tables | `st.dataframe(df)` | -| `st.data_editor()` | Editable tables | `edited = st.data_editor(df)` | -| `st.metric()` | KPI displays | `st.metric("Sales", "$1M", "+10%")` | -| `st.json()` | JSON display | `st.json({"key": "value"})` | -| `st.code()` | Code blocks | `st.code("print('hello')")` | - -### Layout Components - -| Component | Use Case | Example | -|-----------|----------|---------| -| `st.columns()` | Side-by-side | `col1, col2 = st.columns(2)` | -| `st.expander()` | Collapsible | `with st.expander("Details"):` | -| `st.tabs()` | Tabbed interface | `tab1, tab2 = st.tabs(["A", "B"])` | -| `st.container()` | Grouping | `with st.container():` | -| `st.empty()` | Placeholder | `placeholder = st.empty()` | - -### Status Components - -| Component | Use Case | Example | -|-----------|----------|---------| -| `st.success()` | Success message | `st.success("Saved!")` | -| `st.error()` | Error message | `st.error("Failed!")` | -| `st.warning()` | Warning message | `st.warning("Caution!")` | -| `st.info()` | Info message | `st.info("Note: ...")` | -| `st.spinner()` | Loading indicator | `with st.spinner("Loading..."):` | -| `st.progress()` | Progress bar | `st.progress(0.5)` | - ---- - -## Common Pitfalls - -### 1. Page Config Not First - -**❌ Wrong:** -```python -import streamlit as st -st.title("My App") -st.set_page_config(...) # Error! -``` - -**✅ Correct:** -```python -import streamlit as st -st.set_page_config(...) # Must be first! -st.title("My App") -``` - -### 2. Not Caching Connections - -**❌ Wrong:** -```python -def get_data(): - conn = sql.connect(...) # Creates new connection every rerun! - return conn.cursor().fetchall() -``` - -**✅ Correct:** -```python -@st.cache_resource -def get_connection(): - return sql.connect(...) - -def get_data(): - conn = get_connection() # Reuses cached connection - return conn.cursor().fetchall() -``` - -### 3. Expensive Operations in Main Flow - -**❌ Wrong:** -```python -# Runs on every rerun! -data = expensive_api_call() -processed_data = expensive_processing(data) -``` - -**✅ Correct:** -```python -@st.cache_data(ttl=300) -def get_processed_data(): - data = expensive_api_call() - return expensive_processing(data) - -data = get_processed_data() # Cached for 5 minutes -``` - -### 4. Not Using Forms for Multiple Inputs - -**❌ Wrong:** -```python -# Page reruns on EVERY input change -name = st.text_input("Name") # Rerun -email = st.text_input("Email") # Rerun -phone = st.text_input("Phone") # Rerun -``` - -**✅ Correct:** -```python -# Page only reruns on submit -with st.form("user_form"): - name = st.text_input("Name") - email = st.text_input("Email") - phone = st.text_input("Phone") - submitted = st.form_submit_button("Submit") -``` - -### 5. Modifying State During Render - -**❌ Wrong:** -```python -if st.button("Increment"): - st.session_state.count += 1 # ❌ Can cause issues - st.write(st.session_state.count) # May not update immediately -``` - -**✅ Correct:** -```python -if 'count' not in st.session_state: - st.session_state.count = 0 - -if st.button("Increment"): - st.session_state.count += 1 - -st.write(f"Count: {st.session_state.count}") # Display outside callback -``` - ---- - -## Performance Optimization - -### 1. Use Appropriate Caching - -```python -# For connections (persist across sessions) -@st.cache_resource -def get_db_connection(): - return sql.connect(...) - -# For data (serialize and cache with TTL) -@st.cache_data(ttl=600) -def load_orders(): - return fetch_orders() -``` - -### 2. Lazy Loading - -```python -# Don't load all data upfront -def load_page(): - if page == "Dashboard": - load_dashboard_data() # Only load what's needed - elif page == "Orders": - load_orders_data() -``` - -### 3. Pagination - -```python -# Don't display 10,000 rows at once -page_size = 50 -page_num = st.number_input("Page", min_value=1, value=1) - -start_idx = (page_num - 1) * page_size -end_idx = start_idx + page_size - -st.dataframe(df[start_idx:end_idx]) -``` - -### 4. Debouncing with Forms - -```python -# Use forms to prevent reruns on every keystroke -with st.form("search_form"): - search = st.text_input("Search") - submitted = st.form_submit_button("Search") - -if submitted and search: - results = backend.search(search) - st.dataframe(results) -``` - -### 5. Fragment Updates (Streamlit 1.24+) - -```python -@st.experimental_fragment -def render_chart(): - """Only this fragment reruns, not entire page""" - data = load_chart_data() - st.plotly_chart(create_chart(data)) - -# Main page doesn't rerun when fragment updates -st.title("Dashboard") -render_chart() -``` - ---- - -## Deployment Configuration - -### app.yaml for Databricks - -```yaml -command: - - "streamlit" - - "run" - - "streamlit_app.py" - - "--server.port" - - "8080" - - "--server.address" - - "0.0.0.0" - - "--server.headless" - - "true" - -env: - - name: USE_MOCK_BACKEND - value: "false" - - name: DATABRICKS_WAREHOUSE_ID - value: "your-warehouse-id" - - name: DATABRICKS_CATALOG - value: "main" - - name: DATABRICKS_SCHEMA - value: "app_schema" -``` - -### requirements.txt - -```txt -streamlit>=1.28.0 -pandas>=2.0.0 -plotly>=5.17.0 -databricks-sdk>=0.12.0 -databricks-sql-connector>=3.0.0 -pydantic>=2.0.0 -python-dotenv>=1.0.0 -``` - ---- - -## Reference Resources - -- **[Databricks Streamlit Tutorial](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/tutorial-streamlit)** - Official tutorial -- **[Databricks Apps Cookbook - Streamlit](https://apps-cookbook.dev/docs/category/streamlit/)** - Code examples -- **[Streamlit Read Delta Table](https://apps-cookbook.dev/docs/streamlit/tables/tables_read/)** - Connection patterns -- **[Streamlit Documentation](https://docs.streamlit.io/)** - Full API reference - ---- - -## Key Takeaways - -1. **Always cache resources** - Use `@st.cache_resource` for connections -2. **Page config first** - Must be the first Streamlit command -3. **Use forms** - Prevent reruns for multiple inputs -4. **Session state** - For data persistence across reruns -5. **SDK Config pattern** - For Databricks authentication -6. **Layout wisely** - Columns, expanders, tabs for organization -7. **Handle errors** - Graceful degradation and user feedback - -Streamlit is perfect for rapid development of data-focused applications on Databricks! diff --git a/.claude/skills/asset-bundles/SDP_guidance.md b/.claude/skills/databricks-asset-bundles/SDP_guidance.md similarity index 100% rename from .claude/skills/asset-bundles/SDP_guidance.md rename to .claude/skills/databricks-asset-bundles/SDP_guidance.md diff --git a/.claude/skills/asset-bundles/SKILL.md b/.claude/skills/databricks-asset-bundles/SKILL.md similarity index 89% rename from .claude/skills/asset-bundles/SKILL.md rename to .claude/skills/databricks-asset-bundles/SKILL.md index 02e11d00..4253e8e9 100644 --- a/.claude/skills/asset-bundles/SKILL.md +++ b/.claude/skills/databricks-asset-bundles/SKILL.md @@ -1,5 +1,5 @@ --- -name: asset-bundles +name: databricks-asset-bundles description: "Create and configure Databricks Asset Bundles (DABs) with best practices for multi-environment deployments. Use when working with: (1) Creating new DAB projects, (2) Adding resources (dashboards, pipelines, jobs, alerts), (3) Configuring multi-environment deployments, (4) Setting up permissions, (5) Deploying or running bundle resources" --- @@ -61,6 +61,8 @@ targets: ### Dashboard Resources +**Support for dataset_catalog and dataset_schema parameters added in Databricks CLI 0.281.0 (January 2026)** + ```yaml resources: dashboards: @@ -68,6 +70,8 @@ resources: display_name: "[${bundle.target}] Dashboard Title" file_path: ../src/dashboards/dashboard.lvdash.json # Relative to resources/ warehouse_id: ${var.warehouse_id} + dataset_catalog: ${var.catalog} # Default catalog used by all datasets in the dashboard if not otherwise specified in the query + dataset_schema: ${var.schema} # Default schema used by all datasets in the dashboard if not otherwise specified in the query permissions: - level: CAN_RUN group_name: "users" @@ -289,7 +293,7 @@ databricks bundle destroy -t prod --auto-approve | **Catalog doesn't exist** | Create catalog first or update variable | | **"admins" group error on jobs** | Cannot modify admins permissions on jobs | | **Volume permissions** | Use `grants` not `permissions` for volumes | -| **Hardcoded catalog in dashboard** | Create environment-specific files or parameterize JSON | +| **Hardcoded catalog in dashboard** | Use dataset_catalog parameter (CLI v0.281.0+), create environment-specific files, or parameterize JSON | | **App not starting after deploy** | Apps require `databricks bundle run ` to start | | **App env vars not working** | Environment variables go in `app.yaml` (source dir), not databricks.yml | | **Wrong app source path** | Use `../` from resources/ dir if source is in project root | @@ -303,6 +307,14 @@ databricks bundle destroy -t prod --auto-approve 4. **Groups**: Use `"users"` for all workspace users 5. **Job permissions**: Verify custom groups exist; can't modify "admins" +## Related Skills + +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - pipeline definitions referenced by DABs +- **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - app deployment via DABs +- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python app deployment via DABs +- **[databricks-config](../databricks-config/SKILL.md)** - profile and authentication setup for CLI/SDK +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - job orchestration managed through bundles + ## Resources - [Databricks Asset Bundles Documentation](https://docs.databricks.com/dev-tools/bundles/) diff --git a/.claude/skills/asset-bundles/alerts_guidance.md b/.claude/skills/databricks-asset-bundles/alerts_guidance.md similarity index 100% rename from .claude/skills/asset-bundles/alerts_guidance.md rename to .claude/skills/databricks-asset-bundles/alerts_guidance.md diff --git a/.claude/skills/databricks-config/SKILL.md b/.claude/skills/databricks-config/SKILL.md index 12952293..2053f152 100644 --- a/.claude/skills/databricks-config/SKILL.md +++ b/.claude/skills/databricks-config/SKILL.md @@ -72,3 +72,10 @@ cluster_id = 1217-064531-c9c3ngyn View full configuration at: ~/.databrickscfg ``` + +## Related Skills + +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - uses profiles configured by this skill +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - references workspace profiles for deployment targets +- **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - apps that connect via configured profiles +- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python apps using configured profiles diff --git a/.claude/skills/databricks-dbsql/SKILL.md b/.claude/skills/databricks-dbsql/SKILL.md new file mode 100644 index 00000000..24bf2694 --- /dev/null +++ b/.claude/skills/databricks-dbsql/SKILL.md @@ -0,0 +1,300 @@ +--- +name: databricks-dbsql +description: >- + Databricks SQL (DBSQL) advanced features and SQL warehouse capabilities. + This skill MUST be invoked when the user mentions: "DBSQL", "Databricks SQL", + "SQL warehouse", "SQL scripting", "stored procedure", "CALL procedure", + "materialized view", "CREATE MATERIALIZED VIEW", "pipe syntax", "|>", + "geospatial", "H3", "ST_", "spatial SQL", "collation", "COLLATE", + "ai_query", "ai_classify", "ai_extract", "ai_gen", "AI function", + "http_request", "remote_query", "read_files", "Lakehouse Federation", + "recursive CTE", "WITH RECURSIVE", "multi-statement transaction", + "temp table", "temporary view", "pipe operator". + SHOULD also invoke when the user asks about SQL best practices, data modeling + patterns, or advanced SQL features on Databricks. +--- + +# Databricks SQL (DBSQL) - Advanced Features + +## Quick Reference + +| Feature | Key Syntax | Since | Reference | +|---------|-----------|-------|-----------| +| SQL Scripting | `BEGIN...END`, `DECLARE`, `IF/WHILE/FOR` | DBR 16.3+ | [sql-scripting.md](sql-scripting.md) | +| Stored Procedures | `CREATE PROCEDURE`, `CALL` | DBR 17.0+ | [sql-scripting.md](sql-scripting.md) | +| Recursive CTEs | `WITH RECURSIVE` | DBR 17.0+ | [sql-scripting.md](sql-scripting.md) | +| Transactions | `BEGIN ATOMIC...END` | Preview | [sql-scripting.md](sql-scripting.md) | +| Materialized Views | `CREATE MATERIALIZED VIEW` | Pro/Serverless | [materialized-views-pipes.md](materialized-views-pipes.md) | +| Temp Tables | `CREATE TEMPORARY TABLE` | All | [materialized-views-pipes.md](materialized-views-pipes.md) | +| Pipe Syntax | `\|>` operator | DBR 16.1+ | [materialized-views-pipes.md](materialized-views-pipes.md) | +| Geospatial (H3) | `h3_longlatash3()`, `h3_polyfillash3()` | DBR 11.2+ | [geospatial-collations.md](geospatial-collations.md) | +| Geospatial (ST) | `ST_Point()`, `ST_Contains()`, 80+ funcs | DBR 16.0+ | [geospatial-collations.md](geospatial-collations.md) | +| Collations | `COLLATE`, `UTF8_LCASE`, locale-aware | DBR 16.1+ | [geospatial-collations.md](geospatial-collations.md) | +| AI Functions | `ai_query()`, `ai_classify()`, 11+ funcs | DBR 15.1+ | [ai-functions.md](ai-functions.md) | +| http_request | `http_request(conn, ...)` | Pro/Serverless | [ai-functions.md](ai-functions.md) | +| remote_query | `SELECT * FROM remote_query(...)` | Pro/Serverless | [ai-functions.md](ai-functions.md) | +| read_files | `SELECT * FROM read_files(...)` | All | [ai-functions.md](ai-functions.md) | +| Data Modeling | Star schema, Liquid Clustering | All | [best-practices.md](best-practices.md) | + +--- + +## Common Patterns + +### SQL Scripting - Procedural ETL + +```sql +BEGIN + DECLARE v_count INT; + DECLARE v_status STRING DEFAULT 'pending'; + + SET v_count = (SELECT COUNT(*) FROM catalog.schema.raw_orders WHERE status = 'new'); + + IF v_count > 0 THEN + INSERT INTO catalog.schema.processed_orders + SELECT *, current_timestamp() AS processed_at + FROM catalog.schema.raw_orders + WHERE status = 'new'; + + SET v_status = 'completed'; + ELSE + SET v_status = 'skipped'; + END IF; + + SELECT v_status AS result, v_count AS rows_processed; +END +``` + +### Stored Procedure with Error Handling + +```sql +CREATE OR REPLACE PROCEDURE catalog.schema.upsert_customers( + IN p_source STRING, + OUT p_rows_affected INT +) +LANGUAGE SQL +SQL SECURITY INVOKER +BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + SET p_rows_affected = -1; + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = concat('Upsert failed for source: ', p_source); + END; + + MERGE INTO catalog.schema.dim_customer AS t + USING (SELECT * FROM identifier(p_source)) AS s + ON t.customer_id = s.customer_id + WHEN MATCHED THEN UPDATE SET * + WHEN NOT MATCHED THEN INSERT *; + + SET p_rows_affected = (SELECT COUNT(*) FROM identifier(p_source)); +END; + +-- Invoke: +CALL catalog.schema.upsert_customers('catalog.schema.staging_customers', ?); +``` + +### Materialized View with Scheduled Refresh + +```sql +CREATE OR REPLACE MATERIALIZED VIEW catalog.schema.daily_revenue + CLUSTER BY (order_date) + SCHEDULE EVERY 1 HOUR + COMMENT 'Hourly-refreshed daily revenue by region' +AS SELECT + order_date, + region, + SUM(amount) AS total_revenue, + COUNT(DISTINCT customer_id) AS unique_customers +FROM catalog.schema.fact_orders +JOIN catalog.schema.dim_store USING (store_id) +GROUP BY order_date, region; +``` + +### Pipe Syntax - Readable Transformations + +```sql +-- Traditional SQL rewritten with pipe syntax +FROM catalog.schema.fact_orders + |> WHERE order_date >= current_date() - INTERVAL 30 DAYS + |> AGGREGATE SUM(amount) AS total, COUNT(*) AS cnt GROUP BY region, product_category + |> WHERE total > 10000 + |> ORDER BY total DESC + |> LIMIT 20; +``` + +### AI Functions - Enrich Data with LLMs + +```sql +-- Classify support tickets +SELECT + ticket_id, + description, + ai_classify(description, ARRAY('billing', 'technical', 'account', 'feature_request')) AS category, + ai_analyze_sentiment(description) AS sentiment +FROM catalog.schema.support_tickets +LIMIT 100; + +-- Extract entities from text +SELECT + doc_id, + ai_extract(content, ARRAY('person_name', 'company', 'dollar_amount')) AS entities +FROM catalog.schema.contracts; + +-- General-purpose AI query with structured output +SELECT ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + concat('Summarize this customer feedback in JSON with keys: topic, sentiment, action_items. Feedback: ', feedback), + returnType => 'STRUCT>' +) AS analysis +FROM catalog.schema.customer_feedback +LIMIT 50; +``` + +### Geospatial - Proximity Search with H3 + +```sql +-- Find stores within 5km of each customer using H3 indexing +WITH customer_h3 AS ( + SELECT *, h3_longlatash3(longitude, latitude, 7) AS h3_cell + FROM catalog.schema.customers +), +store_h3 AS ( + SELECT *, h3_longlatash3(longitude, latitude, 7) AS h3_cell + FROM catalog.schema.stores +) +SELECT + c.customer_id, + s.store_id, + ST_Distance( + ST_Point(c.longitude, c.latitude), + ST_Point(s.longitude, s.latitude) + ) AS distance_m +FROM customer_h3 c +JOIN store_h3 s ON h3_ischildof(c.h3_cell, h3_toparent(s.h3_cell, 5)) +WHERE ST_Distance( + ST_Point(c.longitude, c.latitude), + ST_Point(s.longitude, s.latitude) +) < 5000; +``` + +### Collation - Case-Insensitive Search + +```sql +-- Create table with case-insensitive collation +CREATE TABLE catalog.schema.products ( + product_id BIGINT GENERATED ALWAYS AS IDENTITY, + name STRING COLLATE UTF8_LCASE, + category STRING COLLATE UTF8_LCASE, + price DECIMAL(10, 2) +); + +-- Queries automatically case-insensitive (no LOWER() needed) +SELECT * FROM catalog.schema.products +WHERE name = 'MacBook Pro'; -- matches 'macbook pro', 'MACBOOK PRO', etc. +``` + +### http_request - Call External APIs + +```sql +-- Set up connection first (one-time) +CREATE CONNECTION my_api_conn + TYPE HTTP + OPTIONS (host 'https://api.example.com', bearer_token secret('scope', 'token')); + +-- Call API from SQL +SELECT + order_id, + http_request( + conn => 'my_api_conn', + method => 'POST', + path => '/v1/validate', + json => to_json(named_struct('order_id', order_id, 'amount', amount)) + ).text AS api_response +FROM catalog.schema.orders +WHERE needs_validation = true; +``` + +### read_files - Ingest Raw Files + +```sql +-- Read JSON files from a Volume with schema hints +SELECT * +FROM read_files( + '/Volumes/catalog/schema/raw/events/', + format => 'json', + schemaHints => 'event_id STRING, timestamp TIMESTAMP, payload MAP', + pathGlobFilter => '*.json', + recursiveFileLookup => true +); + +-- Read CSV with options +SELECT * +FROM read_files( + '/Volumes/catalog/schema/raw/sales/', + format => 'csv', + header => true, + delimiter => '|', + dateFormat => 'yyyy-MM-dd', + schema => 'sale_id INT, sale_date DATE, amount DECIMAL(10,2), store STRING' +); +``` + +### Recursive CTE - Hierarchy Traversal + +```sql +WITH RECURSIVE org_chart AS ( + -- Anchor: top-level managers + SELECT employee_id, name, manager_id, 0 AS depth, ARRAY(name) AS path + FROM catalog.schema.employees + WHERE manager_id IS NULL + + UNION ALL + + -- Recursive: direct reports + SELECT e.employee_id, e.name, e.manager_id, o.depth + 1, array_append(o.path, e.name) + FROM catalog.schema.employees e + JOIN org_chart o ON e.manager_id = o.employee_id + WHERE o.depth < 10 -- safety limit +) +SELECT * FROM org_chart ORDER BY depth, name; +``` + +### remote_query - Federated Queries + +```sql +-- Query PostgreSQL via Lakehouse Federation +SELECT * +FROM remote_query( + 'my_postgres_connection', + database => 'my_database', + query => 'SELECT customer_id, email, created_at FROM customers WHERE active = true' +); +``` + +--- + +## Reference Files + +Load these for detailed syntax, full parameter lists, and advanced patterns: + +| File | Contents | When to Read | +|------|----------|--------------| +| [sql-scripting.md](sql-scripting.md) | SQL Scripting, Stored Procedures, Recursive CTEs, Transactions | User needs procedural SQL, error handling, loops, dynamic SQL | +| [materialized-views-pipes.md](materialized-views-pipes.md) | Materialized Views, Temp Tables/Views, Pipe Syntax | User needs MVs, refresh scheduling, temp objects, pipe operator | +| [geospatial-collations.md](geospatial-collations.md) | 39 H3 functions, 80+ ST functions, Collation types and hierarchy | User needs spatial analysis, H3 indexing, case/accent handling | +| [ai-functions.md](ai-functions.md) | 13 AI functions, http_request, remote_query, read_files (all options) | User needs AI enrichment, API calls, federation, file ingestion | +| [best-practices.md](best-practices.md) | Data modeling, performance, Liquid Clustering, anti-patterns | User needs architecture guidance, optimization, or modeling advice | + +--- + +## Key Guidelines + +- **Always use Serverless SQL warehouses** for AI functions, MVs, and http_request +- **Use `LIMIT` during development** with AI functions to control costs +- **Prefer Liquid Clustering over partitioning** for new tables (1-4 keys max) +- **Use `CLUSTER BY AUTO`** when unsure about clustering keys +- **Star schema in Gold layer** for BI; OBT acceptable in Silver +- **Define PK/FK constraints** on dimensional models for query optimization +- **Use `COLLATE UTF8_LCASE`** for user-facing string columns that need case-insensitive search +- **Use MCP tools** (`execute_sql`, `execute_sql_multi`) to test and validate all SQL before deploying diff --git a/.claude/skills/databricks-dbsql/ai-functions.md b/.claude/skills/databricks-dbsql/ai-functions.md new file mode 100644 index 00000000..0853c6bb --- /dev/null +++ b/.claude/skills/databricks-dbsql/ai-functions.md @@ -0,0 +1,1348 @@ +# AI Functions, http_request, remote_query, and read_files Reference + +Comprehensive reference for Databricks SQL advanced functions: built-in AI functions, HTTP requests, Lakehouse Federation remote queries, and file reading. + +--- + +## Table of Contents + +- [AI Functions Overview](#ai-functions-overview) +- [ai_query -- General-Purpose AI Function](#ai_query----general-purpose-ai-function) +- [Task-Specific AI Functions](#task-specific-ai-functions) + - [ai_gen](#ai_gen) + - [ai_classify](#ai_classify) + - [ai_extract](#ai_extract) + - [ai_analyze_sentiment](#ai_analyze_sentiment) + - [ai_similarity](#ai_similarity) + - [ai_summarize](#ai_summarize) + - [ai_translate](#ai_translate) + - [ai_fix_grammar](#ai_fix_grammar) + - [ai_mask](#ai_mask) +- [Document and Multimodal AI Functions](#document-and-multimodal-ai-functions) + - [ai_parse_document](#ai_parse_document) +- [Time Series AI Functions](#time-series-ai-functions) + - [ai_forecast](#ai_forecast) +- [Vector Search Function](#vector-search-function) + - [vector_search](#vector_search) +- [http_request Function](#http_request-function) +- [remote_query Function (Lakehouse Federation)](#remote_query-function-lakehouse-federation) +- [read_files Table-Valued Function](#read_files-table-valued-function) + +--- + +## AI Functions Overview + +Databricks AI Functions are built-in SQL functions that invoke state-of-the-art generative AI models directly from SQL. They run on Databricks Foundation Model APIs and are available from Databricks SQL, notebooks, Lakeflow Spark Declarative Pipelines, and Workflows. + +**Common Requirements for All AI Functions:** +- Workspace must be in a region supporting AI Functions optimized for batch inference +- Not available on Databricks SQL Classic (requires Serverless SQL Warehouse) +- Databricks Runtime 15.1+ for notebooks; 15.4 ML LTS recommended for batch workloads +- Models licensed under Apache 2.0 or LLAMA 3.3 Community License +- Currently tuned for English (underlying models support multiple languages) +- Public Preview, HIPAA compliant + +**Rate Limits and Billing:** +- AI Functions are subject to Foundation Model API rate limits +- Billed as Databricks SQL compute plus token usage on Foundation Model APIs +- Use `LIMIT` in queries during development to control costs + +--- + +## ai_query -- General-Purpose AI Function + +The most powerful and flexible AI function. Queries any serving endpoint (Foundation Models, external models, or custom ML models) for real-time or batch inference. + +### Syntax + +```sql +-- Basic invocation +ai_query(endpoint, request) + +-- Full invocation with all optional parameters +ai_query( + endpoint, + request, + returnType => type_expression, + failOnError => boolean, + modelParameters => named_struct(...), + responseFormat => format_string, + files => content_expression +) +``` + +### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `endpoint` | STRING | Yes | Name of a Foundation Model, external model, or custom model serving endpoint in the same workspace | +| `request` | STRING or STRUCT | Yes | For LLM endpoints: STRING prompt. For custom ML endpoints: single column or STRUCT matching expected input features | +| `returnType` | Expression | No | Expected return type (DDL-style). Optional in Runtime 15.2+; required in 15.1 and below | +| `failOnError` | BOOLEAN | No | Default `true`. When `false`, returns STRUCT with `response` and `errorStatus` fields instead of failing | +| `modelParameters` | STRUCT | No | Model parameters via `named_struct()` (Runtime 15.3+) | +| `responseFormat` | STRING | No | Controls output format: `'text'`, `'json_object'`, or a DDL/JSON schema string (Runtime 15.4 LTS+, chat models only) | +| `files` | Expression | No | Multimodal file input for image processing (JPEG, PNG supported) | + +### Return Types + +| Scenario | Return Type | +|----------|-------------| +| `failOnError => true` (default) | Parsed response matching endpoint type or `returnType` | +| `failOnError => false` | `STRUCT` where T is the parsed type | +| With `responseFormat` | Structured output matching the specified schema | + +### Model Parameters + +```sql +-- Control generation with modelParameters +SELECT ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + 'Explain quantum computing in 3 sentences.', + modelParameters => named_struct( + 'max_tokens', 256, + 'temperature', 0.1, + 'top_p', 0.9 + ) +) AS response; +``` + +Common model parameters: +- `max_tokens` (INT) -- Maximum tokens to generate +- `temperature` (DOUBLE) -- Randomness (0.0 = deterministic, 2.0 = max random) +- `top_p` (DOUBLE) -- Nucleus sampling threshold +- `stop` (ARRAY) -- Stop sequences + +### Structured Output with responseFormat + +> **Note:** The top-level `responseFormat` STRUCT must contain exactly one field. To return multiple fields, wrap them in a single outer field. + +```sql +-- Force JSON output matching a schema (top-level STRUCT must have exactly one field) +SELECT ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + 'Extract the product name, price, and category from: "Sony WH-1000XM5 headphones, $348, Electronics"', + responseFormat => 'STRUCT>' +) AS extracted; +``` + +### Batch Inference on Tables + +```sql +-- Classify all rows in a table +SELECT + review_id, + review_text, + ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + CONCAT('Classify the following review as positive, negative, or neutral: ', review_text), + responseFormat => 'STRUCT>' + ) AS classification +FROM catalog.schema.product_reviews; +``` + +### Custom ML Model Inference + +```sql +-- Query a custom sklearn/MLflow model +SELECT ai_query( + endpoint => 'spam-classification-endpoint', + request => named_struct( + 'text', email_body, + 'subject', email_subject + ), + returnType => 'BOOLEAN' +) AS is_spam +FROM catalog.schema.inbox_messages; +``` + +### Multimodal (Image) Input + +```sql +-- Analyze images using a vision model +SELECT ai_query( + 'databricks-meta-llama-3-2-90b-instruct', + 'Describe the contents of this image.', + files => READ_FILES('/Volumes/catalog/schema/images/photo.jpg', format => 'binaryFile') +) AS description; +``` + +### Error Handling with failOnError + +```sql +-- Graceful error handling for batch processing +SELECT + id, + result.result AS answer, + result.errorMessage AS error +FROM ( + SELECT + id, + ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + question, + failOnError => false + ) AS result + FROM catalog.schema.questions +); +``` + +### Embedding Generation + +```sql +-- Generate embeddings using ai_query +SELECT + text, + ai_query('databricks-gte-large-en', text) AS embedding +FROM catalog.schema.documents; +``` + +--- + +## Task-Specific AI Functions + +These functions provide simplified, single-purpose interfaces that do not require specifying an endpoint or model. + +### ai_gen + +Generate text from a prompt. + +```sql +ai_gen(prompt) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `prompt` | STRING | The user's request/prompt | + +**Returns:** STRING + +```sql +-- Simple generation +SELECT ai_gen('Generate a concise, cheerful email title for a summer bike sale with 20% discount'); +-- Returns: "Summer Bike Sale: Grab Your Dream Bike at 20% Off!" + +-- Generation using table data +SELECT + question, + ai_gen('You are a teacher. Answer the students question in 50 words: ' || question) AS answer +FROM catalog.schema.questions +LIMIT 10; +``` + +--- + +### ai_classify + +Classify text into one of the provided labels. + +```sql +ai_classify(content, labels) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `content` | STRING | Text to classify | +| `labels` | ARRAY | Classification options (min 2, max 20 elements) | + +**Returns:** STRING matching one of the labels, or NULL if classification fails. + +```sql +-- Simple classification +SELECT ai_classify('My password is leaked.', ARRAY('urgent', 'not urgent')); +-- Returns: "urgent" + +-- Batch product categorization +SELECT + product_name, + description, + ai_classify(description, ARRAY('clothing', 'shoes', 'accessories', 'furniture')) AS category +FROM catalog.schema.products +LIMIT 100; + +-- Support ticket routing +SELECT + ticket_id, + ai_classify( + description, + ARRAY('billing', 'technical', 'account', 'feature_request', 'other') + ) AS department +FROM catalog.schema.support_tickets; +``` + +--- + +### ai_extract + +Extract named entities from text. + +```sql +ai_extract(content, labels) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `content` | STRING | Text to extract entities from | +| `labels` | ARRAY | Entity types to extract | + +**Returns:** STRUCT where each field corresponds to a label, containing the extracted entity as STRING. Returns NULL if content is NULL. + +```sql +-- Extract person, location, organization +SELECT ai_extract( + 'John Doe lives in New York and works for Acme Corp.', + ARRAY('person', 'location', 'organization') +); +-- Returns: {"person": "John Doe", "location": "New York", "organization": "Acme Corp."} + +-- Extract contact details +SELECT ai_extract( + 'Send an email to jane.doe@example.com about the meeting at 10am.', + ARRAY('email', 'time') +); +-- Returns: {"email": "jane.doe@example.com", "time": "10am"} + +-- Batch entity extraction from customer feedback +SELECT + feedback_id, + ai_extract(feedback_text, ARRAY('product', 'issue', 'person')) AS entities +FROM catalog.schema.customer_feedback; +``` + +--- + +### ai_analyze_sentiment + +Perform sentiment analysis on text. + +```sql +ai_analyze_sentiment(content) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `content` | STRING | Text to analyze | + +**Returns:** STRING -- one of `'positive'`, `'negative'`, `'neutral'`, or `'mixed'`. Returns NULL if sentiment cannot be determined. + +```sql +SELECT ai_analyze_sentiment('I am happy'); -- Returns: "positive" +SELECT ai_analyze_sentiment('I am sad'); -- Returns: "negative" +SELECT ai_analyze_sentiment('It is what it is'); -- Returns: "neutral" + +-- Aggregate sentiment by product +SELECT + product_id, + ai_analyze_sentiment(review_text) AS sentiment, + COUNT(*) AS review_count +FROM catalog.schema.reviews +GROUP BY product_id, ai_analyze_sentiment(review_text); +``` + +--- + +### ai_similarity + +Compute semantic similarity between two text strings. + +```sql +ai_similarity(expr1, expr2) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `expr1` | STRING | First text to compare | +| `expr2` | STRING | Second text to compare | + +**Returns:** FLOAT -- Semantic similarity score where 1.0 means identical. The score is relative and should only be used for ranking. + +```sql +-- Exact match +SELECT ai_similarity('Apache Spark', 'Apache Spark'); +-- Returns: 1.0 + +-- Find similar company names (fuzzy matching) +SELECT company_name, ai_similarity(company_name, 'Databricks') AS score +FROM catalog.schema.customers +ORDER BY score DESC +LIMIT 10; + +-- Duplicate detection +SELECT + a.id AS id_a, + b.id AS id_b, + ai_similarity(a.description, b.description) AS similarity +FROM catalog.schema.products a +JOIN catalog.schema.products b ON a.id < b.id +WHERE ai_similarity(a.description, b.description) > 0.85; +``` + +--- + +### ai_summarize + +Generate a summary of text. + +```sql +ai_summarize(content [, max_words]) +``` + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `content` | STRING | Yes | Text to summarize | +| `max_words` | INTEGER | No | Target word count for summary. Default: 50. Set to 0 for no limit | + +**Returns:** STRING. Returns NULL if content is NULL. + +```sql +-- Summarize with default 50-word limit +SELECT ai_summarize( + 'Apache Spark is a unified analytics engine for large-scale data processing. ' + || 'It provides high-level APIs in Java, Scala, Python and R, and an optimized ' + || 'engine that supports general execution graphs.' +); + +-- Summarize with custom word limit +SELECT ai_summarize(article_body, 100) AS summary +FROM catalog.schema.articles; + +-- Executive summaries for reports +SELECT + report_id, + report_title, + ai_summarize(report_body, 30) AS executive_summary +FROM catalog.schema.quarterly_reports; +``` + +--- + +### ai_translate + +Translate text to a target language. + +```sql +ai_translate(content, to_lang) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `content` | STRING | Text to translate | +| `to_lang` | STRING | Target language code | + +**Supported Languages:** English (`en`), German (`de`), French (`fr`), Italian (`it`), Portuguese (`pt`), Hindi (`hi`), Spanish (`es`), Thai (`th`). + +**Returns:** STRING. Returns NULL if content is NULL. + +```sql +-- English to Spanish +SELECT ai_translate('Hello, how are you?', 'es'); +-- Returns: "Hola, como estas?" + +-- Spanish to English +SELECT ai_translate('La vida es un hermoso viaje.', 'en'); +-- Returns: "Life is a beautiful journey." + +-- Translate product descriptions for localization +SELECT + product_id, + description AS original, + ai_translate(description, 'fr') AS french, + ai_translate(description, 'de') AS german +FROM catalog.schema.products; +``` + +--- + +### ai_fix_grammar + +Correct grammatical errors in text. + +```sql +ai_fix_grammar(content) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `content` | STRING | Text to correct | + +**Returns:** STRING with corrected grammar. Returns NULL if content is NULL. + +```sql +SELECT ai_fix_grammar('This sentence have some mistake'); +-- Returns: "This sentence has some mistakes" + +SELECT ai_fix_grammar('She dont know what to did.'); +-- Returns: "She doesn't know what to do." + +-- Clean up user-generated content +SELECT + comment_id, + original_text, + ai_fix_grammar(original_text) AS corrected_text +FROM catalog.schema.user_comments; +``` + +--- + +### ai_mask + +Mask specified entity types in text (PII redaction). + +```sql +ai_mask(content, labels) +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `content` | STRING | Text containing entities to mask | +| `labels` | ARRAY | Entity types to mask (e.g., `'person'`, `'email'`, `'phone'`, `'address'`, `'location'`, `'ssn'`, `'credit_card'`) | + +**Returns:** STRING with specified entities replaced by `[MASKED]`. Returns NULL if content is NULL. + +```sql +-- Mask personal information +SELECT ai_mask( + 'John Doe lives in New York. His email is john.doe@example.com.', + ARRAY('person', 'email') +); +-- Returns: "[MASKED] lives in New York. His email is [MASKED]." + +-- Mask contact details +SELECT ai_mask( + 'Contact me at 555-1234 or visit us at 123 Main St.', + ARRAY('phone', 'address') +); +-- Returns: "Contact me at [MASKED] or visit us at [MASKED]" + +-- Create anonymized dataset +CREATE TABLE catalog.schema.anonymized_feedback AS +SELECT + feedback_id, + ai_mask(feedback_text, ARRAY('person', 'email', 'phone', 'address')) AS masked_text, + category +FROM catalog.schema.customer_feedback; +``` + +--- + +## Document and Multimodal AI Functions + +### ai_parse_document + +Extract structured content from unstructured documents (PDF, DOCX, PPTX, images). + +```sql +ai_parse_document(content) +ai_parse_document(content, options_map) +``` + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `content` | BINARY | Yes | Document as binary blob data | +| `options` | MAP | No | Configuration options | + +**Options Map Keys:** + +| Key | Values | Description | +|-----|--------|-------------| +| `version` | `'2.0'` | Output schema version | +| `imageOutputPath` | Volume path | Path to save rendered page images in Unity Catalog volume | +| `descriptionElementTypes` | `''`, `'figure'`, `'*'` | Controls AI-generated descriptions. Default: `'*'` (all elements) | + +**Returns:** VARIANT with structure: +- `document.pages[]` -- Page metadata (id, image_uri) +- `document.elements[]` -- Extracted content (type, content, bbox, description) +- `error_status[]` -- Error details per page +- `metadata` -- File and schema version info + +**Supported Formats:** PDF, JPG/JPEG, PNG, DOC/DOCX, PPT/PPTX + +**Requirements:** Databricks Runtime 17.1+, US/EU region or cross-geography routing enabled. + +```sql +-- Basic document parsing +SELECT ai_parse_document(content) +FROM READ_FILES('/Volumes/catalog/schema/volume/docs/', format => 'binaryFile'); + +-- Parse with options (save images, version 2.0) +SELECT ai_parse_document( + content, + map( + 'version', '2.0', + 'imageOutputPath', '/Volumes/catalog/schema/volume/images/', + 'descriptionElementTypes', '*' + ) +) +FROM READ_FILES('/Volumes/catalog/schema/volume/invoices/', format => 'binaryFile'); + +-- Parse documents then extract structured data with ai_query +WITH parsed AS ( + SELECT + path, + ai_parse_document(content) AS doc + FROM READ_FILES('/Volumes/catalog/schema/volume/invoices/', format => 'binaryFile') +) +SELECT + path, + ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + CONCAT('Extract vendor name, invoice number, and total from: ', doc:document:elements[0]:content::STRING), + responseFormat => 'STRUCT' + ) AS invoice_data +FROM parsed; +``` + +--- + +## Time Series AI Functions + +### ai_forecast + +Forecast time series data using a built-in prophet-like model. This is a table-valued function (TVF). + +```sql +ai_forecast( + observed TABLE, + horizon DATE | TIMESTAMP | STRING, + time_col STRING, + value_col STRING | ARRAY, + group_col STRING | ARRAY | NULL DEFAULT NULL, + prediction_interval_width DOUBLE DEFAULT 0.95, + frequency STRING DEFAULT 'auto', + seed INTEGER | NULL DEFAULT NULL, + parameters STRING DEFAULT '{}' +) +``` + +### Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `observed` | TABLE | Required | Training data passed as `TABLE(subquery)` or `TABLE(table_name)` | +| `horizon` | DATE/TIMESTAMP/STRING | Required | Right-exclusive forecast end time | +| `time_col` | STRING | Required | Name of DATE or TIMESTAMP column in observed data | +| `value_col` | STRING or ARRAY | Required | One or more numeric columns to forecast | +| `group_col` | STRING, ARRAY, or NULL | NULL | Partition column(s) for independent per-group forecasts | +| `prediction_interval_width` | DOUBLE | 0.95 | Confidence level for prediction bounds (0 to 1) | +| `frequency` | STRING | `'auto'` | Time granularity. Auto-infers from recent data. For DATE columns use: `'day'`, `'week'`, `'month'`. For TIMESTAMP columns: `'D'`, `'W'`, `'M'`, `'H'`, etc. | +| `seed` | INTEGER or NULL | NULL | Random seed for reproducibility | +| `parameters` | STRING | `'{}'` | JSON-encoded advanced settings | + +**Advanced Parameters (JSON):** +- `global_cap` -- Upper bound for logistic growth +- `global_floor` -- Lower bound for logistic growth +- `daily_order` -- Fourier order for daily seasonality +- `weekly_order` -- Fourier order for weekly seasonality + +### Return Columns + +For each `value_col` named `v`, the output contains: +- `{v}_forecast` (DOUBLE) -- Point forecast +- `{v}_upper` (DOUBLE) -- Upper prediction bound +- `{v}_lower` (DOUBLE) -- Lower prediction bound +- Plus the original time column and group columns + +**Requirements:** Serverless SQL Warehouse. + +```sql +-- Basic revenue forecast +SELECT * FROM ai_forecast( + TABLE(SELECT ds, revenue FROM catalog.schema.daily_sales), + horizon => '2025-12-31', + time_col => 'ds', + value_col => 'revenue' +); + +-- Multi-metric forecast by group +SELECT * FROM ai_forecast( + TABLE( + SELECT date, zipcode, revenue, trip_count + FROM catalog.schema.regional_metrics + ), + horizon => '2025-06-30', + time_col => 'date', + value_col => ARRAY('revenue', 'trip_count'), + group_col => 'zipcode', + prediction_interval_width => 0.90, + frequency => 'D' +); + +-- Monthly forecast with growth constraints (use 'month' for DATE columns, not 'M') +SELECT * FROM ai_forecast( + TABLE(catalog.schema.monthly_kpis), + horizon => '2026-01-01', + time_col => 'month', + value_col => 'active_users', + frequency => 'month', + parameters => '{"global_floor": 0}' +); +``` + +--- + +## Vector Search Function + +### vector_search + +Query a Mosaic AI Vector Search index using SQL. This is a table-valued function. + +```sql +-- Databricks Runtime 15.3+ +SELECT * FROM vector_search( + index => index_name, + query_text => search_text, -- OR query_vector => embedding_array + num_results => max_results, + query_type => 'ANN' | 'HYBRID' +) +``` + +### Parameters (Named Arguments Required) + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `index` | STRING constant | Required | Fully qualified name of the vector search index | +| `query_text` | STRING | -- | Search string (for Delta Sync indexes with embedding source) | +| `query_vector` | ARRAY | -- | Pre-computed embedding vector to search | +| `num_results` | INTEGER | 10 | Max records returned (max 100) | +| `query_type` | STRING | `'ANN'` | `'ANN'` for approximate nearest neighbor, `'HYBRID'` for hybrid search | + +**Returns:** Table containing all index columns with top matching records. + +**Requirements:** Serverless SQL Warehouse, Select permission on the index. + +```sql +-- Text-based similarity search +SELECT * FROM vector_search( + index => 'catalog.schema.product_index', + query_text => 'wireless noise canceling headphones', + num_results => 5 +); + +-- Hybrid search (combines keyword + semantic) +SELECT * FROM vector_search( + index => 'catalog.schema.support_docs_index', + query_text => 'Wi-Fi connection issues with router model LMP-9R2', + query_type => 'HYBRID', + num_results => 3 +); + +-- Vector-based search with pre-computed embedding +SELECT * FROM vector_search( + index => 'catalog.schema.embeddings_index', + query_vector => ARRAY(0.45, -0.35, 0.78, 0.22), + num_results => 10 +); + +-- Batch search using LATERAL join +SELECT + q.query_text, + q.query_id, + results.* +FROM catalog.schema.search_queries q, +LATERAL ( + SELECT * FROM vector_search( + index => 'catalog.schema.knowledge_base_index', + query_text => q.query_text, + num_results => 3 + ) +) AS results; +``` + +--- + +## http_request Function + +Make HTTP requests to external services from SQL using Unity Catalog HTTP connections. + +### Syntax + +```sql +http_request( + CONN => connection_name, + METHOD => http_method, + PATH => path, + HEADERS => header_map, + PARAMS => param_map, + JSON => json_body +) +``` + +### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `CONN` | STRING constant | Yes | Name of an existing HTTP connection | +| `METHOD` | STRING constant | Yes | HTTP method: `'GET'`, `'POST'`, `'PUT'`, `'DELETE'`, `'PATCH'` | +| `PATH` | STRING constant | Yes | Path appended to the connection's base_path. Cannot contain directory traversal (`../`) | +| `HEADERS` | MAP | No | Request headers. Default: NULL | +| `PARAMS` | MAP | No | Query parameters. Default: NULL | +| `JSON` | STRING expression | No | Request body as JSON string | + +### Return Type + +`STRUCT` +- `status_code` -- HTTP response status (e.g., 200, 403, 404) +- `text` -- Response body (typically JSON) + +**Requirements:** Databricks Runtime 16.2+, Unity Catalog enabled workspace, USE CONNECTION privilege. + +### Creating HTTP Connections + +```sql +-- Bearer token authentication +CREATE CONNECTION slack_conn TYPE HTTP +OPTIONS ( + host 'https://slack.com', + port '443', + base_path '/api/', + bearer_token secret('my-scope', 'slack-token') +); + +-- OAuth Machine-to-Machine +CREATE CONNECTION github_conn TYPE HTTP +OPTIONS ( + host 'https://api.github.com', + port '443', + base_path '/', + client_id secret('my-scope', 'github-client-id'), + client_secret secret('my-scope', 'github-client-secret'), + oauth_scope 'repo read:org', + token_endpoint 'https://github.com/login/oauth/access_token' +); +``` + +**Connection Options:** + +| Option | Type | Description | +|--------|------|-------------| +| `host` | STRING | Base URL of the external service | +| `port` | STRING | Network port (typically `'443'` for HTTPS) | +| `base_path` | STRING | Root path for API endpoints | +| `bearer_token` | STRING | Auth token (use `secret()` for security) | +| `client_id` | STRING | OAuth application identifier | +| `client_secret` | STRING | OAuth application secret | +| `oauth_scope` | STRING | Space-delimited OAuth scopes | +| `token_endpoint` | STRING | OAuth token endpoint URL | +| `authorization_endpoint` | STRING | OAuth authorization redirect URL | +| `oauth_credential_exchange_method` | STRING | `'header_and_body'`, `'body_only'`, or `'header_only'` | + +### Examples + +```sql +-- POST a Slack message +SELECT http_request( + CONN => 'slack_conn', + METHOD => 'POST', + PATH => '/chat.postMessage', + JSON => to_json(named_struct('channel', '#alerts', 'text', 'Pipeline completed successfully')) +); + +-- GET request with headers and params +SELECT http_request( + CONN => 'github_conn', + METHOD => 'GET', + PATH => '/repos/databricks/spark/issues', + HEADERS => map('Accept', 'application/vnd.github+json'), + PARAMS => map('state', 'open', 'per_page', '5') +); + +-- Parse JSON response +SELECT + response.status_code, + from_json(response.text, 'STRUCT') AS issue +FROM ( + SELECT http_request( + CONN => 'github_conn', + METHOD => 'GET', + PATH => '/repos/databricks/spark/issues/1' + ) AS response +); + +-- Webhook notification triggered by data changes +SELECT http_request( + CONN => 'webhook_conn', + METHOD => 'POST', + PATH => '/notify', + JSON => to_json(named_struct( + 'event', 'data_quality_alert', + 'table', 'catalog.schema.orders', + 'message', CONCAT('Null rate exceeded threshold: ', CAST(null_pct AS STRING)) + )) +) +FROM catalog.schema.data_quality_metrics +WHERE null_pct > 0.05; +``` + +--- + +## remote_query Function (Lakehouse Federation) + +Run SQL queries against external databases using their native SQL syntax, returning results as a table in Databricks SQL. This is a table-valued function. + +### Overview + +Lakehouse Federation enables querying external databases without migrating data. It supports two modes: +- **Query Federation** -- Queries are pushed down to external databases via JDBC +- **Catalog Federation** -- Queries access foreign tables directly in object storage + +### Syntax + +```sql +SELECT * FROM remote_query( + '', + => '' + [, ...] +) +``` + +### Supported Databases + +| Database | Connection Type | +|----------|----------------| +| PostgreSQL | `POSTGRESQL` | +| MySQL | `MYSQL` | +| Microsoft SQL Server | `SQLSERVER` | +| Oracle | `ORACLE` | +| Teradata | `TERADATA` | +| Amazon Redshift | `REDSHIFT` | +| Snowflake | `SNOWFLAKE` | +| Google BigQuery | `BIGQUERY` | +| Databricks | `DATABRICKS` | + +### Parameters by Database Type + +**PostgreSQL / MySQL / SQL Server / Redshift / Teradata:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `database` | STRING | Yes | Remote database name | +| `query` | STRING | One of query/dbtable | SQL query in the remote database's native syntax | +| `dbtable` | STRING | One of query/dbtable | Fully qualified table name | +| `fetchsize` | STRING | No | Number of rows to fetch per round trip | +| `partitionColumn` | STRING | No | Column used for parallel read partitioning | +| `lowerBound` | STRING | No | Lower bound for partition column | +| `upperBound` | STRING | No | Upper bound for partition column | +| `numPartitions` | STRING | No | Number of parallel partitions | + +**Oracle (uses `service_name` instead of `database`):** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `service_name` | STRING | Yes | Oracle service name | +| `query` or `dbtable` | STRING | Yes (one required) | Query or table reference | + +**Snowflake:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `database` | STRING | Yes | Snowflake database | +| `schema` | STRING | No | Schema name (defaults to `public`) | +| `query` or `dbtable` | STRING | Yes (one required) | Query or table reference | +| `query_timeout` | STRING | No | Query timeout in seconds | +| `partition_size_in_mb` | STRING | No | Partition size for reads | + +**BigQuery:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `query` or `dbtable` | STRING | Yes (one required) | Query or table reference | +| `materializationDataset` | STRING | For views/complex queries | Dataset for materialization | +| `materializationProject` | STRING | No | GCP project for materialization | +| `parentProject` | STRING | No | Parent GCP project | + +### Pushdown Control + +| Option | Default | Description | +|--------|---------|-------------| +| `pushdown.limit.enabled` | `true` | Push LIMIT to remote | +| `pushdown.offset.enabled` | `true` | Push OFFSET to remote | +| `pushdown.filters.enabled` | `true` | Push WHERE filters to remote | +| `pushdown.aggregates.enabled` | `true` | Push aggregations to remote | +| `pushdown.sortLimit.enabled` | `true` | Push ORDER BY + LIMIT to remote | + +### Requirements + +- Unity Catalog enabled workspace +- Databricks Runtime 17.3+ (clusters) or SQL Warehouse 2025.35+ (Pro/Serverless) +- Network connectivity to target database +- `USE CONNECTION` privilege or `SELECT` on a wrapping view + +### Limitations + +- **Read-only**: Only SELECT queries supported (no INSERT, UPDATE, DELETE, MERGE, DDL, or stored procedures) + +### Creating Connections + +```sql +-- PostgreSQL connection +CREATE CONNECTION my_postgres TYPE POSTGRESQL +OPTIONS ( + host 'pg-server.example.com', + port '5432', + user secret('my-scope', 'pg-user'), + password secret('my-scope', 'pg-password') +); + +-- SQL Server connection +CREATE CONNECTION my_sqlserver TYPE SQLSERVER +OPTIONS ( + host 'sql-server.example.com', + port '1433', + user secret('my-scope', 'sql-user'), + password secret('my-scope', 'sql-password') +); +``` + +### Examples + +```sql +-- Basic query against PostgreSQL +SELECT * FROM remote_query( + 'my_postgres', + database => 'sales_db', + query => 'SELECT customer_id, name, email FROM customers WHERE active = true' +); + +-- Parallel read from SQL Server +SELECT * FROM remote_query( + 'my_sqlserver', + database => 'orders_db', + dbtable => 'dbo.transactions', + partitionColumn => 'transaction_id', + lowerBound => '0', + upperBound => '1000000', + numPartitions => '10' +); + +-- Join federated data with local Delta tables +SELECT + o.order_id, + o.amount, + c.name, + c.email +FROM catalog.schema.orders o +JOIN remote_query( + 'my_postgres', + database => 'crm_db', + query => 'SELECT customer_id, name, email FROM customers' +) c ON o.customer_id = c.customer_id; + +-- Access delegation via view +CREATE VIEW catalog.schema.federated_customers AS +SELECT * FROM remote_query( + 'my_postgres', + database => 'crm_db', + query => 'SELECT customer_id, name, region FROM customers' +); + +-- Users only need SELECT on the view, not USE CONNECTION +GRANT SELECT ON VIEW catalog.schema.federated_customers TO `analysts`; +``` + +--- + +## read_files Table-Valued Function + +Read files from cloud storage or Unity Catalog volumes directly in SQL, with automatic format detection and schema inference. + +### Syntax + +```sql +SELECT * FROM read_files( + path + [, option_key => option_value ] [...] +) +``` + +### Core Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `path` | STRING | Yes | URI of data location. Supports `s3://`, `abfss://`, `gs://`, `/Volumes/...` paths. Accepts glob patterns | + +### Common Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `format` | STRING | Auto-detected | File format: `'csv'`, `'json'`, `'parquet'`, `'avro'`, `'orc'`, `'text'`, `'binaryFile'`, `'xml'` | +| `schema` | STRING | Inferred | Explicit schema definition in DDL format | +| `schemaHints` | STRING | None | Override subset of inferred schema columns | +| `rescuedDataColumn` | STRING | `'_rescued_data'` | Column name for data that could not be parsed. Set to empty string to disable | +| `pathGlobFilter` / `fileNamePattern` | STRING | None | Glob pattern to filter files (e.g., `'*.csv'`) | +| `recursiveFileLookup` | BOOLEAN | `false` | Search nested directories | +| `modifiedAfter` | TIMESTAMP STRING | None | Only read files modified after this timestamp | +| `modifiedBefore` | TIMESTAMP STRING | None | Only read files modified before this timestamp | +| `partitionColumns` | STRING | Auto-detected | Comma-separated Hive-style partition columns. Empty string ignores all partitions | +| `useStrictGlobber` | BOOLEAN | `true` | Strict glob pattern matching | +| `inferColumnTypes` | BOOLEAN | `true` | Infer exact column types (vs treating all as STRING) | +| `schemaEvolutionMode` | STRING | -- | Schema evolution behavior: `'none'` to drop rescued data column | + +### CSV-Specific Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `sep` / `delimiter` | STRING | `','` | Field delimiter | +| `header` | BOOLEAN | `false` | First row contains column names | +| `encoding` | STRING | `'UTF-8'` | Character encoding | +| `quote` | STRING | `'"'` | Quote character | +| `escape` | STRING | `'\'` | Escape character | +| `nullValue` | STRING | `''` | String representation of null | +| `dateFormat` | STRING | `'yyyy-MM-dd'` | Date parsing format | +| `timestampFormat` | STRING | `'yyyy-MM-dd\'T\'HH:mm:ss...'` | Timestamp parsing format | +| `mode` | STRING | `'PERMISSIVE'` | Parse mode: `'PERMISSIVE'`, `'DROPMALFORMED'`, `'FAILFAST'` | +| `multiLine` | BOOLEAN | `false` | Allow records spanning multiple lines | +| `ignoreLeadingWhiteSpace` | BOOLEAN | `false` | Trim leading whitespace | +| `ignoreTrailingWhiteSpace` | BOOLEAN | `false` | Trim trailing whitespace | +| `comment` | STRING | None | Line comment character | +| `maxCharsPerColumn` | INTEGER | None | Max characters per column | +| `maxColumns` | INTEGER | None | Max number of columns | +| `mergeSchema` | BOOLEAN | `false` | Merge schemas across files | +| `enforceSchema` | BOOLEAN | `true` | Enforce specified schema | +| `locale` | STRING | `'US'` | Locale for number/date parsing | +| `charToEscapeQuoteEscaping` | STRING | None | Character to escape the quote escape character | +| `readerCaseSensitive` | BOOLEAN | `true` | Case-sensitive column name matching | + +### JSON-Specific Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `multiLine` | BOOLEAN | `false` | Parse multi-line JSON records | +| `allowComments` | BOOLEAN | `false` | Allow Java/C++ style comments | +| `allowSingleQuotes` | BOOLEAN | `true` | Allow single quotes for strings | +| `allowUnquotedFieldNames` | BOOLEAN | `false` | Allow unquoted field names | +| `allowBackslashEscapingAnyCharacter` | BOOLEAN | `false` | Allow backslash to escape any character | +| `allowNonNumericNumbers` | BOOLEAN | `true` | Allow NaN, Infinity, -Infinity | +| `encoding` | STRING | `'UTF-8'` | Character encoding | +| `dateFormat` | STRING | `'yyyy-MM-dd'` | Date parsing format | +| `timestampFormat` | STRING | -- | Timestamp parsing format | +| `inferTimestamp` | BOOLEAN | `false` | Infer timestamp types | +| `prefersDecimal` | BOOLEAN | `false` | Prefer DECIMAL over DOUBLE | +| `primitivesAsString` | BOOLEAN | `false` | Infer all primitives as STRING | +| `singleVariantColumn` | STRING | None | Read entire JSON as single VARIANT column | +| `locale` | STRING | `'US'` | Locale for parsing | +| `mode` | STRING | `'PERMISSIVE'` | Parse mode | +| `readerCaseSensitive` | BOOLEAN | `true` | Case-sensitive column matching | +| `timeZone` | STRING | Session timezone | Timezone for timestamp parsing | + +### XML-Specific Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `rowTag` | STRING | **Required** | XML tag that delimits rows | +| `attributePrefix` | STRING | `'_'` | Prefix for XML attributes | +| `valueTag` | STRING | `'_VALUE'` | Tag for element text content | +| `encoding` | STRING | `'UTF-8'` | Character encoding | +| `ignoreSurroundingSpaces` | BOOLEAN | `true` | Ignore whitespace around values | +| `ignoreNamespace` | BOOLEAN | `false` | Ignore XML namespaces | +| `mode` | STRING | `'PERMISSIVE'` | Parse mode | +| `dateFormat` | STRING | `'yyyy-MM-dd'` | Date parsing format | +| `timestampFormat` | STRING | -- | Timestamp parsing format | +| `locale` | STRING | `'US'` | Locale for parsing | +| `readerCaseSensitive` | BOOLEAN | `true` | Case-sensitive matching | +| `samplingRatio` | DOUBLE | `1.0` | Fraction of rows to sample for schema inference | + +### Parquet / Avro / ORC Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `mergeSchema` | BOOLEAN | `false` | Merge schemas across files | +| `readerCaseSensitive` | BOOLEAN | `true` | Case-sensitive column matching | +| `rescuedDataColumn` | STRING | -- | Column for rescued data | +| `datetimeRebaseMode` | STRING | -- | Rebase mode for datetime values | +| `int96RebaseMode` | STRING | -- | Rebase mode for INT96 timestamps (Parquet only) | + +### Streaming Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `includeExistingFiles` | BOOLEAN | `true` | Process existing files on first run | +| `maxFilesPerTrigger` | INTEGER | None | Max files per micro-batch | +| `maxBytesPerTrigger` | STRING | None | Max bytes per micro-batch | +| `allowOverwrites` | BOOLEAN | `false` | Allow processing of overwritten files | +| `schemaEvolutionMode` | STRING | -- | Schema evolution behavior | +| `schemaLocation` | STRING | -- | Location to store inferred schema | + +### Requirements + +- Databricks Runtime 13.3 LTS and above +- Databricks SQL + +### Examples + +```sql +-- Auto-detect format and schema from cloud storage +SELECT * FROM read_files('s3://my-bucket/data/'); + +-- Read CSV with explicit schema +SELECT * FROM read_files( + '/Volumes/catalog/schema/volume/sales.csv', + format => 'csv', + header => true, + schema => 'order_id INT, customer_id INT, amount DOUBLE, order_date DATE' +); + +-- Read CSV with schema hints (override specific columns only) +SELECT * FROM read_files( + '/Volumes/catalog/schema/volume/events/', + format => 'csv', + header => true, + schemaHints => 'event_timestamp TIMESTAMP, amount DECIMAL(10,2)' +); + +-- Read JSON with multi-line support +SELECT * FROM read_files( + '/Volumes/catalog/schema/volume/api_responses/', + format => 'json', + multiLine => true +); + +-- Read Parquet with merged schema across files +SELECT * FROM read_files( + 's3://my-bucket/parquet-data/', + format => 'parquet', + mergeSchema => true +); + +-- Read XML with row tag +SELECT * FROM read_files( + '/Volumes/catalog/schema/volume/feed.xml', + format => 'xml', + rowTag => 'record' +); + +-- Read binary files (images, PDFs) for ai_parse_document +SELECT path, content FROM read_files( + '/Volumes/catalog/schema/volume/documents/', + format => 'binaryFile' +); + +-- Filter files by glob pattern and modification date +SELECT * FROM read_files( + 's3://my-bucket/logs/', + format => 'json', + pathGlobFilter => '*.json', + modifiedAfter => '2025-01-01T00:00:00Z', + modifiedBefore => '2025-02-01T00:00:00Z' +); + +-- Recursive directory scan with partition discovery +SELECT * FROM read_files( + '/Volumes/catalog/schema/volume/partitioned_data/', + recursiveFileLookup => true, + partitionColumns => 'year,month' +); + +-- Include file metadata +SELECT *, _metadata.file_path, _metadata.file_name, _metadata.file_size +FROM read_files('/Volumes/catalog/schema/volume/data/'); + +-- Create table from files +CREATE TABLE catalog.schema.imported_data AS +SELECT * FROM read_files( + '/Volumes/catalog/schema/volume/export.csv', + format => 'csv', + header => true +); + +-- Streaming table from cloud storage +CREATE STREAMING TABLE catalog.schema.streaming_events AS +SELECT * FROM STREAM read_files( + 's3://my-bucket/events/', + format => 'json', + includeExistingFiles => false, + maxFilesPerTrigger => 100 +); + +-- Read single VARIANT column for semi-structured JSON +SELECT * FROM read_files( + '/Volumes/catalog/schema/volume/complex.json', + format => 'json', + singleVariantColumn => 'raw_data' +); +``` + +--- + +## Combining Functions -- Production Patterns + +### AI-Enhanced ETL Pipeline + +```sql +-- Process customer feedback with multiple AI functions +CREATE OR REPLACE TABLE catalog.schema.enriched_feedback AS +SELECT + feedback_id, + feedback_text, + ai_analyze_sentiment(feedback_text) AS sentiment, + ai_classify(feedback_text, ARRAY('product', 'service', 'billing', 'other')) AS category, + ai_extract(feedback_text, ARRAY('product', 'issue')) AS entities, + ai_summarize(feedback_text, 20) AS summary, + ai_mask(feedback_text, ARRAY('person', 'email', 'phone')) AS anonymized_text +FROM catalog.schema.raw_feedback; +``` + +### Document Processing Pipeline + +```sql +-- Ingest, parse, and query documents +WITH raw_docs AS ( + SELECT path, content + FROM read_files('/Volumes/catalog/schema/volume/contracts/', format => 'binaryFile') +), +parsed AS ( + SELECT path, ai_parse_document(content, map('version', '2.0')) AS doc + FROM raw_docs +) +SELECT + path, + ai_query( + 'databricks-meta-llama-3-3-70b-instruct', + CONCAT('Extract the contract parties, effective date, and termination clause from: ', + doc:document:elements[0]:content::STRING), + responseFormat => 'STRUCT' + ) AS contract_info +FROM parsed; +``` + +### External API Integration with http_request + +```sql +-- Enrich data by calling an external API and joining results +SELECT + o.order_id, + o.tracking_number, + from_json( + tracking.text, + 'STRUCT' + ) AS tracking_info +FROM catalog.schema.orders o +CROSS JOIN LATERAL ( + SELECT http_request( + CONN => 'shipping_api_conn', + METHOD => 'GET', + PATH => CONCAT('/track/', o.tracking_number) + ) AS response +) tracking +WHERE tracking.response.status_code = 200; +``` + +### Federated Analytics + +```sql +-- Combine remote database data with local lakehouse data and AI +SELECT + remote_orders.customer_id, + remote_orders.total_spend, + local_profiles.segment, + ai_classify( + CONCAT('Customer spent $', CAST(remote_orders.total_spend AS STRING), + ' in segment ', local_profiles.segment), + ARRAY('high_value', 'medium_value', 'low_value', 'at_risk') + ) AS value_tier +FROM remote_query( + 'my_postgres', + database => 'sales_db', + query => 'SELECT customer_id, SUM(amount) as total_spend FROM orders GROUP BY customer_id' +) remote_orders +JOIN catalog.schema.customer_profiles local_profiles + ON remote_orders.customer_id = local_profiles.customer_id; +``` diff --git a/.claude/skills/databricks-dbsql/best-practices.md b/.claude/skills/databricks-dbsql/best-practices.md new file mode 100644 index 00000000..a33cdc2a --- /dev/null +++ b/.claude/skills/databricks-dbsql/best-practices.md @@ -0,0 +1,475 @@ +# Data Modeling and DBSQL Best Practices + +Comprehensive reference for data modeling patterns, DBSQL performance optimization, and operational best practices on the Databricks Lakehouse Platform. + +--- + +## Data Modeling Best Practices + +### Star Schema vs Denormalization in the Lakehouse + +The Databricks Lakehouse fully supports dimensional modeling. Star schemas translate well to Delta tables and often deliver superior performance compared to fully denormalized approaches. + +**Star Schema (Dimensional Modeling):** +- Central fact table linked to multiple denormalized dimension tables +- Optimizes for complex analytics and multi-dimensional aggregations +- Provides intuitive business process mapping and scales well with SCDs +- Supports up to ~10 filtering dimensions (5 tables x 2 clustering keys each) +- Clear separation of concerns enables fine-grained governance + +**One Big Table (OBT):** +- Single wide table with all attributes pre-joined +- Eliminates joins, simpler governance (one table to manage) +- Liquid Clustering limited to 1-4 keys, so effective filtering is limited to 1-3 dimensions +- Full table scans become bottlenecks as data grows +- Lacks structured business process mapping +- Complicates fine-grained access controls and data quality checks + +**Key finding:** In benchmarks, dimensional models outperformed OBT (2.6s vs 3.5s) despite requiring joins, because fewer files needed to be scanned. However, with Liquid Clustering applied, OBT achieved >3x improvement (down to 1.13s). Both approaches achieve sub-500ms with automatic caching. + +**Recommended approach:** Use a hybrid medallion architecture: +- Silver layer: OBT or Data Vault for rapid integration and cleansing +- Gold layer: Star schema dimensional models as the curated, business-ready presentation layer for BI and reporting + +### When to Normalize vs Denormalize + +| Use Case | Approach | +|---|---| +| Gold layer for BI reporting | Star schema (denormalized dimensions, normalized facts) | +| Silver layer data integration | Normalized or Data Vault | +| Single-use IoT/logging analytics | OBT (filter by 1-3 dimensions) | +| Multi-dimensional business analysis | Star schema | +| Rapidly evolving schemas | OBT in Silver, star schema in Gold | +| High-cardinality filtering (5+ dimensions) | Star schema with Liquid Clustering per table | + +**Rule of thumb:** Dimension tables should be highly denormalized (flatten many-to-one relationships within a single dimension table). Fact tables should remain normalized at the grain of the business event. + +### Kimball-Style Modeling in Databricks + +Kimball dimensional modeling is the recommended approach for the Gold layer in the Lakehouse: + +1. **Identify the business process** (sales, orders, shipments) +2. **Declare the grain** (one row per transaction, per day, etc.) +3. **Choose dimensions** (who, what, where, when, why, how) +4. **Identify facts** (measurable numeric values at the declared grain) + +**Databricks-specific implementation details:** +- Use Unity Catalog for organizing dimensional models (catalog.schema.table) +- Define PRIMARY KEY constraints on dimension surrogate keys +- Define FOREIGN KEY constraints on fact table dimension keys for query optimization +- Add COMMENT on all tables and columns for discoverability +- Apply TAGS for governance (e.g., PII tagging) to enable downstream AI/BI capabilities +- Use `ANALYZE TABLE ... COMPUTE STATISTICS FOR COLUMNS` on dimension keys to support Adaptive Query Execution + +**Key principle:** "The better you model your data upfront, the more easily you can leverage AI on top of it out of the box." Proper schema design enables downstream AI/BI capabilities. + +### Fact Table Patterns + +**Design rules:** +- Store quantitative, numeric measures at the most granular transactional level +- Use DECIMAL instead of floating-point numbers for financial data +- Include foreign keys referencing dimension tables +- Include degenerate dimensions (source-system identifiers like order numbers) +- Transactional fact tables are typically not updated or versioned +- Cluster fact tables by foreign keys to frequently joined dimensions + +**Types of fact tables:** +- **Transaction facts:** One row per event (most common) +- **Periodic snapshot facts:** One row per entity per time period +- **Accumulating snapshot facts:** One row per entity lifecycle, updated as milestones are reached + +**Fact table Liquid Clustering strategy:** +```sql +CREATE TABLE gold.sales.fact_orders ( + order_key BIGINT GENERATED ALWAYS AS IDENTITY, + customer_key BIGINT NOT NULL, + product_key BIGINT NOT NULL, + date_key INT NOT NULL, + order_amount DECIMAL(18,2), + quantity INT, + CONSTRAINT fk_customer FOREIGN KEY (customer_key) REFERENCES gold.sales.dim_customer(customer_key), + CONSTRAINT fk_product FOREIGN KEY (product_key) REFERENCES gold.sales.dim_product(product_key) +) +CLUSTER BY (date_key, customer_key); +``` + +### Dimension Table Patterns + +**Design rules:** +- Use `GENERATED ALWAYS AS IDENTITY` or hash values for surrogate keys +- Prefer integer surrogate keys over strings for join performance +- Highly denormalize: flatten many-to-one relationships within a single dimension table +- Support complex types: MAP for extensibility, STRUCT for nested attributes, ARRAY for multi-valued attributes +- Avoid using ARRAY/MAP columns as filter predicates (they lack column-level statistics for data skipping) +- Cluster dimension tables by primary key plus common filter columns + +**Dimension table example:** +```sql +CREATE TABLE gold.sales.dim_customer ( + customer_key BIGINT GENERATED ALWAYS AS IDENTITY, + customer_id STRING NOT NULL COMMENT 'Natural key from source system', + full_name STRING, + email STRING, + city STRING, + state STRING, + country STRING, + segment STRING, + effective_start_date TIMESTAMP, + effective_end_date TIMESTAMP, + is_current BOOLEAN, + CONSTRAINT pk_customer PRIMARY KEY (customer_key) +) +CLUSTER BY (customer_key, segment) +COMMENT 'Customer dimension with SCD Type 2 history tracking'; +``` + +### Slowly Changing Dimensions (SCD) Patterns + +**SCD Type 1 (Overwrite):** +- In-place updates without tracking history +- Use MERGE INTO with matched UPDATE +- Suitable for corrections or attributes where history is not needed + +**SCD Type 2 (History Tracking):** +- Version records with surrogate keys and metadata columns +- Include `effective_start_date`, `effective_end_date`, and `is_current` columns +- Use MERGE INTO for implementing SCD Type 2 logic in DBSQL + +**SCD Type 2 with MERGE:** +```sql +MERGE INTO gold.sales.dim_customer AS target +USING ( + SELECT * FROM silver.crm.customers_changes +) AS source +ON target.customer_id = source.customer_id AND target.is_current = TRUE +WHEN MATCHED AND ( + target.full_name != source.full_name OR + target.city != source.city +) THEN UPDATE SET + effective_end_date = current_timestamp(), + is_current = FALSE +WHEN NOT MATCHED THEN INSERT ( + customer_id, full_name, email, city, state, country, segment, + effective_start_date, effective_end_date, is_current +) VALUES ( + source.customer_id, source.full_name, source.email, + source.city, source.state, source.country, source.segment, + current_timestamp(), NULL, TRUE +); +-- Then insert new versions for changed records in a second pass +``` + +**Delta Lake Time Travel** enables historical data access within configured log retention periods as a complementary feature to SCD. + +### Partitioning Strategies + +**Databricks recommends Liquid Clustering over traditional partitioning for all new tables.** + +Traditional partitioning rules of thumb (when needed): +- Keep partition count under 10,000 (ideally under 5,000 distinct values) +- Each partition should contain at least 1 GB of data +- Partition by low-cardinality columns that are frequently used in WHERE clauses (e.g., date, region) +- Works best for highly selective single-partition queries (e.g., filter on one day) + +**When traditional partitioning may still be appropriate:** +- Very large tables (hundreds of terabytes) with a clear, stable partition key +- Queries consistently filter on the same low-cardinality column +- Data lifecycle management requires partition-level operations + +### Liquid Clustering vs Traditional Partitioning + +**Liquid Clustering is the default recommendation for all new Delta tables**, including streaming tables and materialized views. It replaces both partitioning and Z-ORDER. + +| Aspect | Liquid Clustering | Partitioning + Z-ORDER | +|---|---|---| +| Column flexibility | Change clustering keys anytime | Partition column fixed at creation | +| Maintenance | Incremental, automatic with predictive optimization | Manual OPTIMIZE + Z-ORDER required | +| Filter dimensions | Best with 1-4 clustering keys | One partition key + Z-ORDER columns | +| Write overhead | Minimal (only unclustered ZCubes reorganized) | Z-ORDER reorganizes entire table/partition | +| Best for | Most workloads, evolving access patterns | Very large tables with stable, low-cardinality filter | +| Performance | 30-60% query speed improvement for variable queries | Better for single-partition lookup queries | + +**Liquid Clustering key selection best practices:** +- Choose columns most frequently used in query filters and joins +- Limit to 1-4 keys (fewer is better for smaller tables under 10 TB) +- For fact tables: cluster by the most commonly filtered foreign keys +- For dimension tables: cluster by primary key + common filter columns +- Too many keys dilute data skipping benefits; for tables under 10 TB, 2 keys often outperform 4 + +**Important:** Liquid Clustering is not compatible with partitioning or Z-ORDER on the same table. + +### Z-Ordering Considerations + +Z-ORDER is the legacy approach, now superseded by Liquid Clustering: + +- Z-ORDER reorganizes the entire table/partition during optimization (heavier writes) +- Does not track ZCube IDs, so every OPTIMIZE re-sorts all data +- Better suited for read-heavy workloads where write overhead is acceptable +- For new tables, always prefer Liquid Clustering + +**Migration path:** When migrating existing partitioned + Z-ORDERed tables to Liquid Clustering: +1. Drop the partition specification +2. Enable Liquid Clustering with chosen keys +3. Run OPTIMIZE to incrementally cluster data +4. Allow predictive optimization to maintain layout going forward + +--- + +## DBSQL Performance + +### Query Optimization Tips + +**Engine-level optimizations (automatic in DBSQL Serverless):** +- **Predictive Query Execution (PQE):** Monitors tasks in real time, dynamically adjusts query execution to avoid skew, spills, and unnecessary work. Unlike Adaptive Query Execution (AQE) which re-plans only after a stage completes, PQE detects issues like data skew or memory spills as they occur and replans immediately. +- **Photon Vectorized Shuffle:** Keeps data in compact columnar format, sorts within CPU cache, and uses vectorized instructions for 1.5x higher shuffle throughput. Best for CPU-bound workloads (large joins, wide aggregations). +- **Low Shuffle Merge:** Optimized MERGE implementation that reduces shuffle overhead for most common workloads. + +**Manual optimization actions:** +- Run `ANALYZE TABLE ... COMPUTE STATISTICS FOR COLUMNS` on dimension keys and frequently filtered columns to support AQE and data skipping +- Set `'delta.dataSkippingStatsColumns'` table property to specify which columns collect statistics +- Define PRIMARY KEY and FOREIGN KEY constraints to help the query optimizer +- Use deterministic queries (avoid `NOW()`, `CURRENT_TIMESTAMP()` in filters) to benefit from query result caching +- Prefer `CREATE OR REPLACE TABLE` over delete-then-create patterns +- Use `DECIMAL` over `FLOAT`/`DOUBLE` for financial calculations + +**SQL writing tips for DBSQL:** +- Filter early, aggregate late: push WHERE clauses as close to the source as possible +- Prefer explicit column lists over SELECT * +- Use CTEs for readability but be aware the optimizer may inline them +- Avoid Python/Scala UDFs when native SQL functions exist (UDFs require serialization between Python and Spark, significantly slowing queries) +- Use window functions instead of self-joins where possible +- Leverage QUALIFY clause for row-level filtering after window functions + +### Warehouse Sizing Guidance + +**Databricks recommends serverless SQL warehouses for most workloads.** Serverless uses Intelligent Workload Management (IWM) to automatically manage query workloads. + +**Sizing strategy:** +- Start with a single larger warehouse and let serverless features manage concurrency +- Size down if needed rather than starting small and scaling up +- If queries spill to disk, increase the cluster size + +**Scaling configuration:** +- Low concurrency (1-2 queries): keep max_clusters low +- Unpredictable spikes: set max_num_clusters high with target_utilization ~70% +- For dashboards with variable/infrequent load: enable aggressive auto-scaling and auto-stopping + +**Serverless advantages:** +- Start and scale up in seconds +- Scale down earlier than non-serverless warehouses +- Pay only when queries are running +- 30-60 second cold start latency (savings from no idle time far outweigh this) +- All 2025 optimizations (PQE, Photon Vectorized Shuffle) are automatically available + +### Caching Strategies + +**Query Result Cache:** +- DBSQL caches results per-cluster for all queries +- Cache is invalidated when underlying Delta data changes +- To maximize cache hits, use deterministic queries (no `NOW()`, `RAND()`, etc.) +- Both OBT and star schema achieve sub-500ms with automatic caching after first run + +**Delta Cache (Disk Cache):** +- Automatically caches remote data on local SSD in columnar format +- Accelerates data reads without manual configuration on serverless warehouses +- Particularly effective for repeated scans of the same tables + +**Best practice:** Design dashboards and reports to use parameterized queries that hit the same underlying patterns, maximizing cache reuse. + +### Photon Engine Benefits + +Photon is a vectorized query engine written in C++ that runs natively on Databricks: + +- Enabled by default on all DBSQL serverless warehouses +- Processes data in columnar batches using CPU vector instructions (SIMD) +- Excels at: large joins, wide aggregations, string processing, data shuffles +- 2025 vectorized shuffle delivers 1.5x higher shuffle throughput +- Combined with PQE, delivers up to 25% faster queries on top of existing 5x gains + +### Recent Performance Improvements (2025) + +| Improvement | Impact | +|---|---| +| Overall production workloads | Up to 40% faster (automatic, no tuning) | +| Photon Vectorized Shuffle | 1.5x higher shuffle throughput | +| PQE + Photon Vectorized Shuffle combined | Up to 25% faster on top of existing 5x gains | +| Spatial SQL queries | Up to 17x faster (R-tree indexing, optimized spatial joins) | +| AI functions | Up to 85x faster for large batch workloads | +| End-to-end Unity Catalog latency | Up to 10x improvement | +| 3-year cumulative improvement | 5x faster across customer workloads | + +All improvements are live in DBSQL Serverless with nothing to enable. + +### Cost Optimization Patterns + +1. **Use serverless SQL warehouses:** Pay only when queries run, auto-scale and auto-stop +2. **Enable predictive optimization:** Automatically runs OPTIMIZE and VACUUM on Unity Catalog managed tables +3. **Right-size warehouses:** Start larger, scale down based on actual usage patterns +4. **Avoid idle warehouses:** Use aggressive auto-stop for dashboards with infrequent load +5. **Leverage caching:** Design deterministic queries to maximize result cache hits +6. **Use Liquid Clustering:** Reduces scan volume, fewer DBUs consumed per query +7. **Collect statistics:** `ANALYZE TABLE` enables better query plans, reducing wasted compute +8. **Monitor with Query Profile:** Identify expensive operations, spills, and skew +9. **Use materialized views** for frequently computed aggregations +10. **Avoid UDFs:** Native functions are dramatically faster, no serialization overhead + +--- + +## Delta Lake Optimization for DBSQL + +### OPTIMIZE, VACUUM, and ANALYZE + +**Recommended execution order:** OPTIMIZE -> VACUUM -> ANALYZE + +**OPTIMIZE:** +- Compacts small files into larger ones (target 1 GB by default) +- Run frequently on tables with many small files (especially after streaming writes) +- Configurable target size via `delta.targetFileSize` table property +- With Liquid Clustering: only reorganizes unclustered ZCubes (incremental) + +**VACUUM:** +- Removes old files no longer in the transaction log +- Reduces storage costs +- Use compute-optimized instances (AWS C5, Azure F-series, GCP C2) +- Default retention: 7 days (configurable via `delta.deletedFileRetentionDuration`) +- Never set retention below the longest-running query duration + +**ANALYZE TABLE:** +- Computes column-level statistics for query optimization +- Run immediately after table overwrites or major data changes +- Focus on columns used in WHERE clauses, JOINs, and GROUP BY + +**Predictive optimization (recommended):** + +> **Note:** On serverless SQL warehouses, `delta.enableOptimizeWrite` and `delta.autoOptimize.autoCompact` are managed automatically and cannot be set manually (they will raise `DELTA_UNKNOWN_CONFIGURATION`). The properties below apply only to classic compute. For serverless, simply enable predictive optimization at the catalog/schema level. + +```sql +-- Classic compute only: +ALTER TABLE catalog.schema.table_name +SET TBLPROPERTIES ('delta.enableOptimizeWrite' = 'true'); +-- For Unity Catalog managed tables, predictive optimization +-- handles OPTIMIZE and VACUUM automatically +``` + +### File Size and Compaction + +- **Auto-compaction:** Combines small files within partitions automatically after writes +- **Optimized writes:** Rebalances data via shuffle before writing to reduce small files +- **Target file size:** Default 1 GB; adjust with `delta.targetFileSize` for specific workloads +- For tables with many small files (streaming ingestion), schedule regular OPTIMIZE jobs + +### Table Properties for Performance + +> **Note:** `delta.enableOptimizeWrite` and `delta.autoOptimize.autoCompact` are only valid on classic compute. On serverless SQL warehouses, these are managed automatically and setting them raises `DELTA_UNKNOWN_CONFIGURATION`. The remaining properties work on both classic and serverless. + +```sql +-- Classic compute only (serverless manages these automatically): +-- 'delta.enableOptimizeWrite' = 'true', +-- 'delta.autoOptimize.autoCompact' = 'true', + +-- Works on both classic and serverless: +ALTER TABLE catalog.schema.my_table SET TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', + 'delta.enableChangeDataFeed' = 'true', + 'delta.deletedFileRetentionDuration' = '30 days', + 'delta.dataSkippingStatsColumns' = 'col1,col2,col3' +); +``` + +--- + +## Unity Catalog Integration Patterns + +### Organization Best Practices + +- Use a three-level namespace: `catalog.schema.table` +- Organize by environment (dev/staging/prod) at the catalog level +- Organize by business domain at the schema level +- Use managed tables (not external) to benefit from predictive optimization and enhanced governance + +### Governance Features for Data Modeling + +- **Primary/Foreign Key constraints:** Inform the query optimizer about table relationships +- **Row filters and column masks:** Fine-grained access control at the table level +- **Tags:** Apply governance tags (e.g., PII, sensitivity level) to tables and columns +- **Comments:** Document all tables and columns for AI/BI discoverability +- **Lineage tracking:** Automatic lineage for understanding data flow through the medallion architecture + +### Entity Relationship Visualization + +Unity Catalog renders entity relationship diagrams when primary and foreign key constraints are defined, providing visual documentation of the dimensional model. + +--- + +## Monitoring and Observability + +- **Query Profile:** Analyze execution plans, identify bottlenecks, spills, and data skew +- **Query History:** Track query performance trends over time +- **Warehouse monitoring:** Track utilization, queue times, and scaling events +- **System tables:** Query `system.billing`, `system.access`, and `system.query` for operational insights +- **Alerts:** Set up SQL alerts for data quality checks and SLA monitoring + +--- + +## Common Anti-Patterns to Avoid + +### Data Modeling Anti-Patterns + +1. **Skipping dimensional modeling in Gold layer:** OBTs are fine for Silver, but Gold should use star schemas for multi-dimensional analysis +2. **Over-partitioning:** More than 5,000-10,000 partitions degrades performance; use Liquid Clustering instead +3. **String surrogate keys:** Use integer IDENTITY columns for better join performance +4. **Missing constraints:** Not defining PK/FK constraints deprives the optimizer of relationship information +5. **Missing comments and tags:** Reduces discoverability for AI/BI tools and governance +6. **Using FLOAT for financial data:** Use DECIMAL to avoid precision errors +7. **Filtering on ARRAY/MAP columns:** These types lack column-level statistics for data skipping + +### Query and Performance Anti-Patterns + +1. **Delete-then-recreate tables:** Use `CREATE OR REPLACE TABLE` instead to preserve time travel and avoid reader interruptions +2. **Python/Scala UDFs when native functions exist:** Serialization overhead dramatically slows queries +3. **Not collecting statistics:** Missing `ANALYZE TABLE` leads to suboptimal query plans +4. **Non-deterministic functions in cached queries:** `NOW()`, `RAND()` etc. prevent query result caching +5. **Partitioning by wrong column:** Partitioning by a column not used in filters causes full scans +6. **Too many Liquid Clustering keys:** For tables under 10 TB, 2 keys often outperform 4 keys +7. **Manual OPTIMIZE/VACUUM without predictive optimization:** Enable predictive optimization for Unity Catalog managed tables + +### Operational Anti-Patterns + +1. **Idle warehouses:** Always enable auto-stop; use serverless for variable workloads +2. **Under-sized warehouses:** Queries spilling to disk waste more DBUs than a larger warehouse +3. **External tables when managed will do:** External tables miss predictive optimization and enhanced governance +4. **Skipping VACUUM:** Unbounded file growth increases storage costs and slows metadata operations +5. **Running VACUUM with too-short retention:** Can break long-running queries and time travel + +--- + +## Quick Reference: SQL Patterns for AI Agents + +When generating SQL for Databricks, prefer these patterns: + +```sql +-- Use CREATE OR REPLACE (not DROP + CREATE) +CREATE OR REPLACE TABLE catalog.schema.my_table AS +SELECT ...; + +-- Use MERGE for upserts (not DELETE + INSERT) +MERGE INTO target USING source +ON target.key = source.key +WHEN MATCHED THEN UPDATE SET ... +WHEN NOT MATCHED THEN INSERT ...; + +-- Use QUALIFY for window function filtering (not subquery) +SELECT *, ROW_NUMBER() OVER (PARTITION BY id ORDER BY ts DESC) AS rn +FROM my_table +QUALIFY rn = 1; + +-- Use DECIMAL for money +SELECT CAST(amount AS DECIMAL(18,2)) AS revenue FROM orders; + +-- Collect statistics after loading +ANALYZE TABLE catalog.schema.my_table COMPUTE STATISTICS FOR ALL COLUMNS; + +-- Enable predictive optimization (classic compute only; serverless manages this automatically) +ALTER TABLE catalog.schema.my_table +SET TBLPROPERTIES ('delta.enableOptimizeWrite' = 'true'); +``` diff --git a/.claude/skills/databricks-dbsql/geospatial-collations.md b/.claude/skills/databricks-dbsql/geospatial-collations.md new file mode 100644 index 00000000..eaa2468f --- /dev/null +++ b/.claude/skills/databricks-dbsql/geospatial-collations.md @@ -0,0 +1,736 @@ +# Geospatial SQL and Collations in Databricks SQL + +--- + +## Part 1: Geospatial SQL + +Databricks SQL provides comprehensive geospatial support through two function families: **H3 functions** for hexagonal grid indexing and **ST functions** for standard spatial operations. Together they enable high-performance geospatial analytics at scale. + +### Geospatial Data Types + +| Type | Description | Coordinate System | SRID Support | +|------|-------------|-------------------|--------------| +| `GEOMETRY` | Spatial objects using Euclidean coordinates (X, Y, optional Z) -- treats Earth as flat | Any projected CRS | 11,000+ SRIDs | +| `GEOGRAPHY` | Geographic objects on Earth's surface using longitude/latitude | WGS 84 | SRID 4326 only | + +**When to use which:** +- Use `GEOMETRY` for projected coordinate systems, Euclidean distance calculations, and when working with local/regional data in meters or feet. +- Use `GEOGRAPHY` for global data using longitude/latitude coordinates and spherical distance calculations. + +### Supported Geometry Subtypes + +Both `GEOMETRY` and `GEOGRAPHY` support: **Point**, **LineString**, **Polygon**, **MultiPoint**, **MultiLineString**, **MultiPolygon**, and **GeometryCollection**. + +### Format Support + +| Format | Description | Import Function | Export Function | +|--------|-------------|-----------------|-----------------| +| WKT | Well-Known Text | `ST_GeomFromWKT`, `ST_GeogFromWKT` | `ST_AsWKT`, `ST_AsText` | +| WKB | Well-Known Binary | `ST_GeomFromWKB`, `ST_GeogFromWKB` | `ST_AsWKB`, `ST_AsBinary` | +| EWKT | Extended WKT (includes SRID) | `ST_GeomFromEWKT`, `ST_GeogFromEWKT` | `ST_AsEWKT` | +| EWKB | Extended WKB (includes SRID) | `ST_GeomFromEWKB` | `ST_AsEWKB` | +| GeoJSON | JSON-based format | `ST_GeomFromGeoJSON`, `ST_GeogFromGeoJSON` | `ST_AsGeoJSON` | +| Geohash | Hierarchical grid encoding | `ST_GeomFromGeoHash`, `ST_PointFromGeoHash` | `ST_GeoHash` | + +--- + +### H3 Geospatial Functions + +H3 is Uber's hexagonal hierarchical spatial index. It divides the Earth into hexagonal cells at 16 resolutions (0-15). Available since Databricks Runtime 11.2 (H3 Java library 3.7.0). No separate installation required. + +#### H3 Import Functions (Coordinate/Geometry to H3) + +| Function | Description | Returns | +|----------|-------------|---------| +| `h3_longlatash3(lon, lat, resolution)` | Convert longitude/latitude to H3 cell ID | `BIGINT` | +| `h3_longlatash3string(lon, lat, resolution)` | Convert longitude/latitude to H3 cell ID | `STRING` (hex) | +| `h3_pointash3(geogExpr, resolution)` | Convert GEOGRAPHY point to H3 cell ID | `BIGINT` | +| `h3_pointash3string(geogExpr, resolution)` | Convert GEOGRAPHY point to H3 cell ID | `STRING` (hex) | +| `h3_polyfillash3(geogExpr, resolution)` | Fill polygon with contained H3 cells | `ARRAY` | +| `h3_polyfillash3string(geogExpr, resolution)` | Fill polygon with contained H3 cells | `ARRAY` | +| `h3_coverash3(geogExpr, resolution)` | Cover geography with minimal set of H3 cells | `ARRAY` | +| `h3_coverash3string(geogExpr, resolution)` | Cover geography with minimal set of H3 cells | `ARRAY` | +| `h3_tessellateaswkb(geogExpr, resolution)` | Tessellate geography using H3 cells | `ARRAY` | +| `h3_try_polyfillash3(geogExpr, resolution)` | Safe polyfill (returns NULL on error) | `ARRAY` | +| `h3_try_polyfillash3string(geogExpr, resolution)` | Safe polyfill (returns NULL on error) | `ARRAY` | +| `h3_try_coverash3(geogExpr, resolution)` | Safe cover (returns NULL on error) | `ARRAY` | +| `h3_try_coverash3string(geogExpr, resolution)` | Safe cover (returns NULL on error) | `ARRAY` | +| `h3_try_tessellateaswkb(geogExpr, resolution)` | Safe tessellate (returns NULL on error) | `ARRAY` | + +#### H3 Export Functions (H3 to Geometry/Format) + +| Function | Description | Returns | +|----------|-------------|---------| +| `h3_boundaryaswkt(h3CellId)` | H3 cell boundary as WKT polygon | `STRING` | +| `h3_boundaryaswkb(h3CellId)` | H3 cell boundary as WKB polygon | `BINARY` | +| `h3_boundaryasgeojson(h3CellId)` | H3 cell boundary as GeoJSON | `STRING` | +| `h3_centeraswkt(h3CellId)` | H3 cell center as WKT point | `STRING` | +| `h3_centeraswkb(h3CellId)` | H3 cell center as WKB point | `BINARY` | +| `h3_centerasgeojson(h3CellId)` | H3 cell center as GeoJSON point | `STRING` | + +#### H3 Conversion Functions + +| Function | Description | +|----------|-------------| +| `h3_h3tostring(h3CellId)` | Convert BIGINT cell ID to hex STRING | +| `h3_stringtoh3(h3CellIdString)` | Convert hex STRING to BIGINT cell ID | + +#### H3 Hierarchy / Traversal Functions + +| Function | Description | +|----------|-------------| +| `h3_resolution(h3CellId)` | Get the resolution of a cell | +| `h3_toparent(h3CellId, resolution)` | Get parent cell at coarser resolution | +| `h3_tochildren(h3CellId, resolution)` | Get all child cells at finer resolution | +| `h3_maxchild(h3CellId, resolution)` | Get child with maximum value | +| `h3_minchild(h3CellId, resolution)` | Get child with minimum value | +| `h3_ischildof(h3CellId1, h3CellId2)` | Test if cell1 is equal to or child of cell2 | + +#### H3 Distance / Neighbor Functions + +| Function | Description | +|----------|-------------| +| `h3_distance(h3CellId1, h3CellId2)` | Grid distance between two cells | +| `h3_try_distance(h3CellId1, h3CellId2)` | Grid distance or NULL if undefined | +| `h3_kring(h3CellId, k)` | All cells within grid distance k (filled disk) | +| `h3_kringdistances(h3CellId, k)` | Cells within distance k with their distances | +| `h3_hexring(h3CellId, k)` | Hollow ring of cells at exactly distance k | + +#### H3 Compaction Functions + +| Function | Description | +|----------|-------------| +| `h3_compact(h3CellIds)` | Compact array of cells to minimal representation | +| `h3_uncompact(h3CellIds, resolution)` | Expand compacted cells to target resolution | + +#### H3 Validation Functions + +| Function | Description | +|----------|-------------| +| `h3_isvalid(expr)` | Check if BIGINT or STRING is valid H3 cell | +| `h3_validate(h3CellId)` | Return cell ID if valid, error otherwise | +| `h3_try_validate(h3CellId)` | Return cell ID if valid, NULL otherwise | +| `h3_ispentagon(h3CellId)` | Check if cell is a pentagon (12 per resolution) | + +#### H3 Examples + +```sql +-- Convert coordinates to H3 cell at resolution 9 +SELECT h3_longlatash3(-73.985428, 40.748817, 9) AS h3_cell; + +-- Index taxi trips by pickup location +CREATE TABLE trips_h3 AS +SELECT + h3_longlatash3(pickup_longitude, pickup_latitude, 12) AS pickup_cell, + h3_longlatash3(dropoff_longitude, dropoff_latitude, 12) AS dropoff_cell, + * +FROM taxi_trips; + +-- Fill zip code polygons with H3 cells for spatial indexing +CREATE TABLE zipcode_h3 AS +SELECT + explode(h3_polyfillash3(geom_wkt, 12)) AS cell, + zipcode, city, state +FROM zipcodes; + +-- Find all trips picked up in a specific zip code using H3 join +SELECT t.* +FROM trips_h3 t +INNER JOIN zipcode_h3 z ON t.pickup_cell = z.cell +WHERE z.zipcode = '10001'; + +-- Proximity search: find all H3 cells within 2 rings of a location +SELECT explode(h3_kring(h3_longlatash3(-73.985, 40.748, 9), 2)) AS nearby_cell; + +-- Aggregate trip counts and get centroids for visualization +SELECT + dropoff_cell, + h3_centerasgeojson(dropoff_cell):coordinates[0] AS lon, + h3_centerasgeojson(dropoff_cell):coordinates[1] AS lat, + count(*) AS trip_count +FROM trips_h3 +GROUP BY dropoff_cell; + +-- Roll up to coarser resolution +SELECT + h3_toparent(pickup_cell, 7) AS parent_cell, + count(*) AS trip_count +FROM trips_h3 +GROUP BY h3_toparent(pickup_cell, 7); + +-- Compact a set of cells for efficient storage +SELECT h3_compact(collect_set(cell)) AS compacted +FROM zipcode_h3 +WHERE zipcode = '10001'; +``` + +--- + +### ST Geospatial Functions + +Native spatial SQL functions operating on `GEOMETRY` and `GEOGRAPHY` types. Requires Databricks Runtime 17.1+. Public Preview. Over 80 functions available. + +#### ST Import Functions (Create Geometry/Geography) + +| Function | Description | Output Type | +|----------|-------------|-------------| +| `ST_GeomFromText(wkt [, srid])` | Create GEOMETRY from WKT | `GEOMETRY` | +| `ST_GeomFromWKT(wkt [, srid])` | Create GEOMETRY from WKT (alias) | `GEOMETRY` | +| `ST_GeomFromWKB(wkb [, srid])` | Create GEOMETRY from WKB | `GEOMETRY` | +| `ST_GeomFromEWKT(ewkt)` | Create GEOMETRY from Extended WKT | `GEOMETRY` | +| `ST_GeomFromEWKB(ewkb)` | Create GEOMETRY from Extended WKB | `GEOMETRY` | +| `ST_GeomFromGeoJSON(geojson)` | Create GEOMETRY(4326) from GeoJSON | `GEOMETRY` | +| `ST_GeomFromGeoHash(geohash)` | Create polygon GEOMETRY from geohash | `GEOMETRY` | +| `ST_GeogFromText(wkt)` | Create GEOGRAPHY(4326) from WKT | `GEOGRAPHY` | +| `ST_GeogFromWKT(wkt)` | Create GEOGRAPHY(4326) from WKT | `GEOGRAPHY` | +| `ST_GeogFromWKB(wkb)` | Create GEOGRAPHY(4326) from WKB | `GEOGRAPHY` | +| `ST_GeogFromEWKT(ewkt)` | Create GEOGRAPHY from Extended WKT | `GEOGRAPHY` | +| `ST_GeogFromGeoJSON(geojson)` | Create GEOGRAPHY(4326) from GeoJSON | `GEOGRAPHY` | +| `ST_Point(x, y [, srid])` | Create point from coordinates | `GEOMETRY` | +| `ST_PointFromGeoHash(geohash)` | Create point from geohash center | `GEOMETRY` | +| `to_geometry(georepExpr)` | Auto-detect format and create GEOMETRY | `GEOMETRY` | +| `to_geography(georepExpr)` | Auto-detect format and create GEOGRAPHY | `GEOGRAPHY` | +| `try_to_geometry(georepExpr)` | Safe geometry creation (NULL on error) | `GEOMETRY` | +| `try_to_geography(georepExpr)` | Safe geography creation (NULL on error) | `GEOGRAPHY` | + +#### ST Export Functions + +| Function | Description | Output | +|----------|-------------|--------| +| `ST_AsText(geo)` | Export as WKT | `STRING` | +| `ST_AsWKT(geo)` | Export as WKT (alias) | `STRING` | +| `ST_AsBinary(geo)` | Export as WKB | `BINARY` | +| `ST_AsWKB(geo)` | Export as WKB (alias) | `BINARY` | +| `ST_AsEWKT(geo)` | Export as Extended WKT | `STRING` | +| `ST_AsEWKB(geo)` | Export as Extended WKB | `BINARY` | +| `ST_AsGeoJSON(geo)` | Export as GeoJSON | `STRING` | +| `ST_GeoHash(geo)` | Export as geohash string | `STRING` | + +#### ST Constructor Functions + +| Function | Description | +|----------|-------------| +| `ST_Point(x, y [, srid])` | Create a point geometry | +| `ST_MakeLine(pointArray)` | Create linestring from array of points | +| `ST_MakePolygon(outer [, innerArray])` | Create polygon from outer ring and optional holes | + +#### ST Accessor Functions + +| Function | Description | Returns | +|----------|-------------|---------| +| `ST_X(geo)` | X coordinate of a point | `DOUBLE` | +| `ST_Y(geo)` | Y coordinate of a point | `DOUBLE` | +| `ST_Z(geo)` | Z coordinate of a point | `DOUBLE` | +| `ST_M(geo)` | M coordinate of a point | `DOUBLE` | +| `ST_XMin(geo)` | Minimum X of bounding box | `DOUBLE` | +| `ST_XMax(geo)` | Maximum X of bounding box | `DOUBLE` | +| `ST_YMin(geo)` | Minimum Y of bounding box | `DOUBLE` | +| `ST_YMax(geo)` | Maximum Y of bounding box | `DOUBLE` | +| `ST_ZMin(geo)` | Minimum Z coordinate | `DOUBLE` | +| `ST_ZMax(geo)` | Maximum Z coordinate | `DOUBLE` | +| `ST_Dimension(geo)` | Topological dimension (0=point, 1=line, 2=polygon) | `INT` | +| `ST_NDims(geo)` | Number of coordinate dimensions | `INT` | +| `ST_NPoints(geo)` | Total number of points | `INT` | +| `ST_NumGeometries(geo)` | Number of geometries in collection | `INT` | +| `ST_NumInteriorRings(geo)` | Number of interior rings (polygon) | `INT` | +| `ST_GeometryType(geo)` | Geometry type as string | `STRING` | +| `ST_GeometryN(geo, n)` | N-th geometry (1-based) from collection | `GEOMETRY` | +| `ST_PointN(geo, n)` | N-th point from linestring | `GEOMETRY` | +| `ST_StartPoint(geo)` | First point of linestring | `GEOMETRY` | +| `ST_EndPoint(geo)` | Last point of linestring | `GEOMETRY` | +| `ST_ExteriorRing(geo)` | Outer ring of polygon | `GEOMETRY` | +| `ST_InteriorRingN(geo, n)` | N-th interior ring of polygon | `GEOMETRY` | +| `ST_Envelope(geo)` | Minimum bounding rectangle | `GEOMETRY` | +| `ST_Envelope_Agg(geo)` | Aggregate: bounding box of all geometries | `GEOMETRY` | +| `ST_Dump(geo)` | Explode multi-geometry into array of singles | `ARRAY` | +| `ST_IsEmpty(geo)` | True if geometry has no points | `BOOLEAN` | + +#### ST Measurement Functions + +| Function | Description | +|----------|-------------| +| `ST_Area(geo)` | Area of a polygon (in CRS units) | +| `ST_Length(geo)` | Length of a linestring (in CRS units) | +| `ST_Perimeter(geo)` | Perimeter of a polygon (in CRS units) | +| `ST_Distance(geo1, geo2)` | Cartesian distance between geometries | +| `ST_DistanceSphere(geo1, geo2)` | Spherical distance in meters (fast, approximate) | +| `ST_DistanceSpheroid(geo1, geo2)` | Geodesic distance in meters on WGS84 (accurate) | +| `ST_Azimuth(geo1, geo2)` | North-based azimuth angle in radians | +| `ST_ClosestPoint(geo1, geo2)` | Point on geo1 closest to geo2 | + +#### ST Topological Relationship Functions (Predicates) + +| Function | Description | +|----------|-------------| +| `ST_Contains(geo1, geo2)` | True if geo1 fully contains geo2 | +| `ST_Within(geo1, geo2)` | True if geo1 is fully within geo2 (inverse of Contains) | +| `ST_Intersects(geo1, geo2)` | True if geometries share any space | +| `ST_Disjoint(geo1, geo2)` | True if geometries share no space | +| `ST_Touches(geo1, geo2)` | True if boundaries touch but interiors do not | +| `ST_Covers(geo1, geo2)` | True if geo1 covers geo2 (no point of geo2 is exterior) | +| `ST_Equals(geo1, geo2)` | True if geometries are topologically equal | +| `ST_DWithin(geo1, geo2, distance)` | True if geometries are within given distance | + +#### ST Overlay Functions (Set Operations) + +| Function | Description | +|----------|-------------| +| `ST_Intersection(geo1, geo2)` | Geometry of shared space | +| `ST_Union(geo1, geo2)` | Geometry combining both inputs | +| `ST_Union_Agg(geo)` | Aggregate: union of all geometries in column | +| `ST_Difference(geo1, geo2)` | Geometry of geo1 minus geo2 | + +#### ST Processing Functions + +| Function | Description | +|----------|-------------| +| `ST_Buffer(geo, radius)` | Expand geometry by radius distance | +| `ST_Centroid(geo)` | Center point of geometry | +| `ST_ConvexHull(geo)` | Smallest convex polygon containing geometry | +| `ST_ConcaveHull(geo, ratio [, allowHoles])` | Concave hull with length ratio | +| `ST_Boundary(geo)` | Boundary of geometry (not available on all SQL Warehouse versions) | +| `ST_Simplify(geo, tolerance)` | Simplify using Douglas-Peucker algorithm | + +#### ST Editor Functions + +| Function | Description | +|----------|-------------| +| `ST_AddPoint(linestring, point [, index])` | Add point to linestring | +| `ST_RemovePoint(linestring, index)` | Remove point from linestring | +| `ST_SetPoint(linestring, index, point)` | Replace point in linestring | +| `ST_FlipCoordinates(geo)` | Swap X and Y coordinates | +| `ST_Multi(geo)` | Convert single geometry to multi-geometry | +| `ST_Reverse(geo)` | Reverse vertex order | + +#### ST Affine Transformation Functions + +| Function | Description | +|----------|-------------| +| `ST_Translate(geo, xOffset, yOffset [, zOffset])` | Move geometry by offset | +| `ST_Scale(geo, xFactor, yFactor [, zFactor])` | Scale geometry by factors | +| `ST_Rotate(geo, angle)` | Rotate geometry around origin (radians) | + +#### ST Spatial Reference System Functions + +| Function | Description | +|----------|-------------| +| `ST_SRID(geo)` | Get SRID of geometry | +| `ST_SetSRID(geo, srid)` | Set SRID value (no reprojection) | +| `ST_Transform(geo, targetSrid)` | Reproject to target coordinate system | + +#### ST Validation + +| Function | Description | +|----------|-------------| +| `ST_IsValid(geo)` | Check if geometry is OGC-valid | + +#### ST Practical Examples + +> **Note:** `GEOMETRY` and `GEOGRAPHY` column types in `CREATE TABLE` require serverless compute with DBR 17.1+. On SQL Warehouses that don't support these column types, use `STRING` columns with WKT representation and convert with `ST_GeomFromText()` / `ST_GeogFromText()` at query time. + +```sql +-- Create a table with geometry columns (requires serverless DBR 17.1+) +CREATE TABLE retail_stores ( + store_id INT, + name STRING, + location GEOMETRY +); + +INSERT INTO retail_stores VALUES + (1, 'Downtown Store', ST_Point(-73.9857, 40.7484, 4326)), + (2, 'Midtown Store', ST_Point(-73.9787, 40.7614, 4326)), + (3, 'Uptown Store', ST_Point(-73.9680, 40.7831, 4326)); + +-- Create delivery zones as polygons +CREATE TABLE delivery_zones ( + zone_id INT, + zone_name STRING, + boundary GEOMETRY +); + +INSERT INTO delivery_zones VALUES + (1, 'Zone A', ST_GeomFromText( + 'POLYGON((-74.00 40.74, -73.97 40.74, -73.97 40.76, -74.00 40.76, -74.00 40.74))', 4326 + )); + +-- Point-in-polygon: find stores within a delivery zone +SELECT s.name, z.zone_name +FROM retail_stores s +JOIN delivery_zones z + ON ST_Contains(z.boundary, s.location); + +-- Distance calculation: find customers within 5km of a store +-- Note: to_geography() expects STRING (WKT/GeoJSON) or BINARY (WKB) input, not GEOMETRY. +-- Use ST_AsText() to convert GEOMETRY to WKT first. +SELECT c.customer_id, c.name, + ST_DistanceSphere(c.location, s.location) AS distance_meters +FROM customers c +CROSS JOIN retail_stores s +WHERE s.store_id = 1 + AND ST_DWithin( + ST_GeogFromText(ST_AsText(c.location)), + ST_GeogFromText(ST_AsText(s.location)), + 5000 -- 5km in meters + ); + +-- Buffer zone: create 1km buffer around a store (use projected CRS for meters) +SELECT ST_Buffer( + ST_Transform(location, 5070), -- project to NAD83/Albers (meters) + 1000 -- 1000 meters +) AS buffer_zone +FROM retail_stores +WHERE store_id = 1; + +-- Area calculation +SELECT zone_name, + ST_Area(ST_Transform(boundary, 5070)) AS area_sq_meters +FROM delivery_zones; + +-- Union of overlapping zones +SELECT ST_Union_Agg(boundary) AS combined_coverage +FROM delivery_zones; + +-- Convert between formats +SELECT + ST_AsText(location) AS wkt, + ST_AsGeoJSON(location) AS geojson, + ST_GeoHash(location) AS geohash +FROM retail_stores; + +-- Spatial join with BROADCAST hint for performance +SELECT /*+ BROADCAST(zones) */ + c.customer_id, z.zone_name +FROM customers c +JOIN delivery_zones zones + ON ST_Contains(zones.boundary, c.location); +``` + +### Combining H3 and ST Functions + +```sql +-- Use H3 for fast pre-filtering, then ST for precise spatial operations +-- Step 1: Index store locations with H3 +CREATE TABLE store_h3 AS +SELECT store_id, name, location, + h3_longlatash3(ST_X(location), ST_Y(location), 9) AS h3_cell +FROM retail_stores; + +-- Step 2: Index customer locations with H3 +CREATE TABLE customer_h3 AS +SELECT customer_id, name, location, + h3_longlatash3(ST_X(location), ST_Y(location), 9) AS h3_cell +FROM customers; + +-- Step 3: Fast proximity using H3 pre-filter + precise ST distance +SELECT s.name AS store, c.name AS customer, + ST_DistanceSphere(s.location, c.location) AS distance_m +FROM store_h3 s +JOIN customer_h3 c + ON c.h3_cell IN (SELECT explode(h3_kring(s.h3_cell, 2))) +WHERE ST_DistanceSphere(s.location, c.location) < 2000; +``` + +### Spatial Join Performance + +Databricks automatically optimizes spatial joins using built-in spatial indexing. Spatial predicates like `ST_Intersects`, `ST_Contains`, and `ST_Within` in JOIN conditions benefit from up to **17x performance improvement** compared to classic clusters. No code changes required -- the optimizer applies spatial indexing automatically. + +**Performance tips:** +- Use `BROADCAST` hint when one side of the join is small enough to fit in memory. +- Use projected coordinate systems (e.g., SRID 5070 in meters) for distance calculations to avoid expensive spheroid functions. +- Combine H3 for coarse pre-filtering with ST for precise operations. +- Use Delta Lake liquid clustering on H3 cell columns for optimized data layout. +- Enable auto-optimization: `delta.autoOptimize.optimizeWrite` and `delta.autoOptimize.autoCompact`. + +--- + +## Part 2: Collations + +Collations define rules for comparing and sorting strings. Databricks supports binary, case-insensitive, accent-insensitive, and locale-specific collations using the ICU library. Available from Databricks Runtime 16.1+. + +### Collation Types + +| Collation | Description | Behavior | +|-----------|-------------|----------| +| `UTF8_BINARY` | Default. Byte-by-byte comparison of UTF-8 encoding | `'A' < 'Z' < 'a'` -- binary order, case/accent sensitive | +| `UTF8_LCASE` | Case-insensitive binary. Converts to lowercase then compares with UTF8_BINARY | `'A' == 'a'` but `'e' != 'e'` (accent sensitive) | +| `UNICODE` | ICU root locale. Language-agnostic Unicode ordering | `'a' < 'A' < 'A' < 'b'` -- groups similar characters | +| Locale-specific | ICU locale-based (e.g., `DE`, `FR`, `JA`) | Language-aware sorting rules | + +### Collation Syntax + +``` +{ UTF8_BINARY | UTF8_LCASE | { UNICODE | locale } [ _ modifier [...] ] } +``` + +Where `locale` is: +``` +language_code [ _ script_code ] [ _ country_code ] +``` + +- `language_code`: ISO 639-1 (e.g., `EN`, `DE`, `FR`, `JA`, `ZH`) +- `script_code`: ISO 15924 (e.g., `Hant` for Traditional Chinese, `Latn` for Latin) +- `country_code`: ISO 3166-1 (e.g., `US`, `DE`, `CAN`) + +### Collation Modifiers (DBR 16.2+) + +| Modifier | Description | Default | +|----------|-------------|---------| +| `CS` | Case-Sensitive: `'A' != 'a'` | Yes (default) | +| `CI` | Case-Insensitive: `'A' == 'a'` | No | +| `AS` | Accent-Sensitive: `'e' != 'e'` | Yes (default) | +| `AI` | Accent-Insensitive: `'e' == 'e'` | No | +| `RTRIM` | Trailing-space insensitive: `'Hello' == 'Hello '` | No | + +Specify at most one from each pair (CS/CI, AS/AI) plus optional RTRIM. Order does not matter. + +### Locale Examples + +| Collation Name | Description | +|----------------|-------------| +| `UNICODE` | ICU root locale, language-agnostic | +| `UNICODE_CI` | Unicode, case-insensitive | +| `UNICODE_CI_AI` | Unicode, case and accent-insensitive | +| `DE` | German sorting rules | +| `DE_CI_AI` | German, case and accent-insensitive | +| `FR_CAN` | French (Canada) | +| `EN_US` | English (United States) | +| `ZH_Hant_MAC` | Traditional Chinese (Macau) | +| `SR` | Serbian (normalized from `SR_CYR_SRN_CS_AS`) | +| `JA` | Japanese | +| `EN_CS_AI` | English, case-sensitive, accent-insensitive | +| `UTF8_LCASE_RTRIM` | Case-insensitive with trailing space trimming | + +### Collation Precedence + +From highest to lowest: + +1. **Explicit** -- Assigned via `COLLATE` expression +2. **Implicit** -- Derived from column, field, or variable definition +3. **Default** -- Applied to string literals and function results +4. **None** -- When combining different implicit collations + +Mixing two different **explicit** collations in the same expression produces an error. + +### Setting Collations at Different Levels + +#### Catalog Level (DBR 17.1+) + +```sql +-- Create catalog with default collation +CREATE CATALOG customer_cat + DEFAULT COLLATION UNICODE_CI_AI; + +-- All schemas, tables, and string columns created in this catalog +-- inherit UNICODE_CI_AI unless overridden +``` + +#### Schema Level (DBR 17.1+) + +```sql +-- Create schema with default collation +CREATE SCHEMA my_schema + DEFAULT COLLATION UNICODE_CI; + +-- Change default collation for new objects (existing objects unchanged) +ALTER SCHEMA my_schema + DEFAULT COLLATION UNICODE_CI_AI; +``` + +#### Table Level (DBR 16.3+) + +```sql +-- Table-level default collation +CREATE TABLE users ( + id INT, + username STRING, -- inherits UNICODE_CI from table default + email STRING, -- inherits UNICODE_CI from table default + password_hash STRING COLLATE UTF8_BINARY -- explicit override +) DEFAULT COLLATION UNICODE_CI; +``` + +#### Column Level (DBR 16.1+) + +```sql +-- Column-level collation +CREATE TABLE products ( + id INT, + name STRING COLLATE UNICODE_CI, + sku STRING COLLATE UTF8_BINARY, + description STRING COLLATE UNICODE_CI_AI +); + +-- Add column with collation +ALTER TABLE products + ADD COLUMN category STRING COLLATE UNICODE_CI; + +-- Change column collation (requires DBR 17.2+; may not be available on all SQL Warehouse versions) +ALTER TABLE products + ALTER COLUMN name SET COLLATION UNICODE_CI_AI; +``` + +#### Expression Level + +```sql +-- Apply collation inline in a query +SELECT * +FROM products +WHERE name COLLATE UNICODE_CI = 'laptop'; + +-- Check the collation of an expression +SELECT collation('test' COLLATE UNICODE_CI); +-- Returns: UNICODE_CI +``` + +### Collation Inheritance Hierarchy + +``` +Catalog DEFAULT COLLATION + -> Schema DEFAULT COLLATION (overrides catalog) + -> Table DEFAULT COLLATION (overrides schema) + -> Column COLLATE (overrides table) + -> Expression COLLATE (overrides column) +``` + +If no collation is specified at any level, `UTF8_BINARY` is used. + +### Collation-Aware String Functions + +Most string functions respect collations. Key collation-aware operations: + +| Function/Operator | Collation Behavior | +|-------------------|-------------------| +| `=`, `!=`, `<`, `>`, `<=`, `>=` | Comparison uses column/expression collation | +| `LIKE` | Pattern matching respects collation | +| `CONTAINS(str, substr)` | Substring search respects collation | +| `STARTSWITH(str, prefix)` | Prefix match respects collation | +| `ENDSWITH(str, suffix)` | Suffix match respects collation | +| `IN (...)` | Membership test respects collation | +| `BETWEEN` | Range comparison respects collation | +| `ORDER BY` | Sorting respects collation | +| `GROUP BY` | Grouping respects collation | +| `DISTINCT` | Deduplication respects collation | +| `REPLACE(str, old, new)` | Search respects collation | +| `TRIM` / `LTRIM` / `RTRIM` | Trim characters respect collation | + +**Performance note:** `STARTSWITH` and `ENDSWITH` with `UTF8_LCASE` collation show up to **10x performance speedup** compared to equivalent `LOWER()` workarounds. + +### Utility Functions + +```sql +-- Get collation of an expression +SELECT collation(name) FROM products; + +-- List all supported collations +SELECT * FROM collations(); + +-- Test collation with COLLATE +SELECT collation('hello' COLLATE DE_CI_AI); +-- Returns: DE_CI_AI +``` + +### Practical Collation Examples + +#### Case-Insensitive Search + +```sql +-- Using column collation (preferred - leverages indexes) +CREATE TABLE users ( + id INT, + username STRING COLLATE UTF8_LCASE, + email STRING COLLATE UTF8_LCASE +); + +INSERT INTO users VALUES + (1, 'JohnDoe', 'John@Example.com'), + (2, 'janedoe', 'JANE@EXAMPLE.COM'); + +-- Case-insensitive match automatically +SELECT * FROM users WHERE username = 'johndoe'; +-- Returns: JohnDoe + +SELECT * FROM users WHERE email = 'john@example.com'; +-- Returns: John@Example.com +``` + +#### Case-Insensitive Search with Expression Collation + +```sql +-- Ad-hoc case-insensitive comparison on a UTF8_BINARY column +SELECT * FROM products +WHERE name COLLATE UNICODE_CI = 'MacBook Pro'; +-- Matches: macbook pro, MACBOOK PRO, MacBook Pro, etc. +``` + +#### Accent-Insensitive Search + +```sql +-- Accent-insensitive matching +CREATE TABLE cities ( + id INT, + name STRING COLLATE UNICODE_CI_AI +); + +INSERT INTO cities VALUES (1, 'Montreal'), (2, 'Montreal'); + +SELECT * FROM cities WHERE name = 'Montreal'; +-- Returns both: Montreal and Montreal (treats e and e as equal) +``` + +#### Locale-Aware Sorting + +```sql +-- German sorting (umlauts sort correctly) +SELECT name +FROM german_customers +ORDER BY name COLLATE DE; +-- Sorts: Arzte before Bauer (A treated as A+e in German sorting) + +-- Swedish sorting (A, A, O sort after Z) +SELECT name +FROM swedish_customers +ORDER BY name COLLATE SV; +``` + +#### Trailing Space Handling + +```sql +-- RTRIM modifier ignores trailing spaces +SELECT 'Hello' COLLATE UTF8_BINARY_RTRIM = 'Hello '; +-- Returns: true + +SELECT 'Hello' COLLATE UTF8_BINARY = 'Hello '; +-- Returns: false +``` + +#### Catalog-Wide Case-Insensitive Setup + +```sql +-- Create a catalog where everything is case-insensitive by default +CREATE CATALOG app_data DEFAULT COLLATION UNICODE_CI; + +USE CATALOG app_data; +CREATE SCHEMA users_schema; +USE SCHEMA users_schema; + +-- All STRING columns automatically use UNICODE_CI +CREATE TABLE accounts ( + id INT, + username STRING, -- UNICODE_CI inherited from catalog + email STRING -- UNICODE_CI inherited from catalog +); + +-- Queries are automatically case-insensitive +SELECT * FROM accounts WHERE username = 'admin'; +-- Matches: Admin, ADMIN, admin, aDmIn, etc. +``` + +### Limitations and Notes + +- `CHECK` constraints and generated column expressions require `UTF8_BINARY` default collation. +- `hive_metastore` catalog tables do not support collation constraints. +- `ALTER SCHEMA ... DEFAULT COLLATION` only affects newly created objects, not existing ones. +- Mixing two different explicit collations in the same expression raises an error. +- `UTF8_LCASE` is used internally for Databricks identifier resolution (catalog, schema, table, column names). +- Databricks normalizes collation names by removing defaults (e.g., `SR_CYR_SRN_CS_AS` simplifies to `SR`). +- Collation modifiers require Databricks Runtime 16.2+. +- Catalog/Schema-level `DEFAULT COLLATION` requires Databricks Runtime 17.1+. diff --git a/.claude/skills/databricks-dbsql/materialized-views-pipes.md b/.claude/skills/databricks-dbsql/materialized-views-pipes.md new file mode 100644 index 00000000..078ad09b --- /dev/null +++ b/.claude/skills/databricks-dbsql/materialized-views-pipes.md @@ -0,0 +1,676 @@ +# Materialized Views, Temporary Tables/Views, and Pipe Syntax + +## 1. Materialized Views in Databricks SQL + +### Overview + +Materialized views (MVs) are Unity Catalog-managed tables that physically store precomputed query results. Unlike standard views that recompute on every query, MVs cache results and update automatically -- either on a schedule, when upstream data changes, or on-demand. + +Key characteristics: +- **Pre-computed storage**: Results are physically stored as Delta tables, reducing query latency +- **Automatic updates**: Changes propagate from source tables via incremental or full refresh +- **Serverless pipelines**: Each MV automatically creates a serverless pipeline for creation and refreshes +- **Incremental refresh**: Can compute only changed data from source tables under certain conditions + +### Requirements + +- **Compute**: Unity Catalog-enabled **Serverless** SQL warehouse +- **Region**: Serverless SQL warehouse support must be available in your region +- **Permissions**: + - Creator needs: `SELECT` on base tables, `USE CATALOG`, `USE SCHEMA`, `CREATE TABLE`, `CREATE MATERIALIZED VIEW` + - Refresh needs: Ownership or `REFRESH` privilege; MV owner must retain `SELECT` on base tables + - Query needs: `SELECT` on the MV, `USE CATALOG`, `USE SCHEMA` + +### CREATE MATERIALIZED VIEW Syntax + +```sql +{ CREATE OR REPLACE MATERIALIZED VIEW | CREATE MATERIALIZED VIEW [ IF NOT EXISTS ] } + view_name + [ column_list ] + [ view_clauses ] + AS query +``` + +**Column list** (optional): +```sql +CREATE MATERIALIZED VIEW mv_name ( + col1 INT NOT NULL, + col2 STRING, + col3 DOUBLE, + CONSTRAINT pk PRIMARY KEY (col1) +) +AS SELECT ... +``` + +**View clauses** (optional): +- `PARTITIONED BY (col1, col2)` -- partition by columns +- `CLUSTER BY (col1, col2)` or `CLUSTER BY AUTO` -- liquid clustering (cannot combine with PARTITIONED BY) +- `COMMENT 'description'` -- view description +- `TBLPROPERTIES ('key' = 'value')` -- user-defined properties +- `WITH ROW FILTER func ON (col1, col2)` -- row-level security +- `MASK func` on columns -- column-level masking +- `SCHEDULE` clause -- automatic refresh schedule +- `TRIGGER ON UPDATE` clause -- event-driven refresh + +### Basic Examples + +```sql +-- Simple materialized view +CREATE MATERIALIZED VIEW catalog.schema.daily_sales + COMMENT 'Daily sales aggregations' +AS SELECT + date, + region, + SUM(sales) AS total_sales, + COUNT(*) AS num_transactions +FROM catalog.schema.raw_sales +GROUP BY date, region; + +-- MV with explicit columns, constraints, and clustering +CREATE MATERIALIZED VIEW catalog.schema.customer_orders ( + customer_id INT NOT NULL, + full_name STRING, + order_count BIGINT, + CONSTRAINT customer_pk PRIMARY KEY (customer_id) +) +CLUSTER BY AUTO +COMMENT 'Customer order counts' +AS SELECT + c.customer_id, + c.full_name, + COUNT(o.order_id) AS order_count +FROM catalog.schema.customers c +INNER JOIN catalog.schema.orders o ON c.customer_id = o.customer_id +GROUP BY c.customer_id, c.full_name; +``` + +### Refresh Options + +MVs support four refresh strategies: + +#### 1. Manual Refresh + +```sql +-- Synchronous (blocks until complete) +REFRESH MATERIALIZED VIEW catalog.schema.daily_sales; + +-- Asynchronous (returns immediately) +REFRESH MATERIALIZED VIEW catalog.schema.daily_sales ASYNC; +``` + +#### 2. Scheduled Refresh (SCHEDULE) + +```sql +-- Every N hours/days/weeks +CREATE OR REPLACE MATERIALIZED VIEW catalog.schema.hourly_metrics + SCHEDULE EVERY 1 HOUR +AS SELECT date_trunc('hour', event_time) AS hour, COUNT(*) AS events +FROM catalog.schema.raw_events +GROUP BY 1; + +-- Cron-based schedule +CREATE OR REPLACE MATERIALIZED VIEW catalog.schema.nightly_report + SCHEDULE CRON '0 0 2 * * ?' AT TIME ZONE 'America/New_York' +AS SELECT * FROM catalog.schema.daily_aggregates; +``` + +Valid intervals: 1-72 hours, 1-31 days, 1-8 weeks. A Databricks Job is automatically created for scheduled refreshes. + +#### 3. Event-Driven Refresh (TRIGGER ON UPDATE) + +Automatically refreshes when upstream data changes: + +```sql +CREATE OR REPLACE MATERIALIZED VIEW catalog.schema.customer_orders + TRIGGER ON UPDATE +AS SELECT c.customer_id, c.name, COUNT(o.order_id) AS order_count +FROM catalog.schema.customers c +JOIN catalog.schema.orders o ON c.customer_id = o.customer_id +GROUP BY c.customer_id, c.name; + +-- With throttle to avoid excessive refreshes +CREATE OR REPLACE MATERIALIZED VIEW catalog.schema.customer_orders + TRIGGER ON UPDATE AT MOST EVERY INTERVAL 5 MINUTES +AS SELECT c.customer_id, c.name, COUNT(o.order_id) AS order_count +FROM catalog.schema.customers c +JOIN catalog.schema.orders o ON c.customer_id = o.customer_id +GROUP BY c.customer_id, c.name; +``` + +Trigger limitations: +- Maximum **10 upstream source tables** and **30 upstream views** +- Minimum **1-minute** interval (default) +- Maximum **1,000** trigger-based MVs per workspace +- Supports Delta tables, managed views, and streaming tables as sources +- Does **not** support Delta Sharing shared tables + +#### 4. Job-Based Orchestration + +Integrate refreshes into existing Databricks Jobs using SQL task types: + +```sql +-- In a Databricks Job SQL task +REFRESH MATERIALIZED VIEW catalog.schema.daily_sales_summary; +``` + +### Managing Schedules After Creation + +```sql +-- Add a schedule to an existing MV +ALTER MATERIALIZED VIEW catalog.schema.my_mv ADD SCHEDULE EVERY 4 HOURS; + +-- Add trigger-based refresh +ALTER MATERIALIZED VIEW catalog.schema.my_mv ADD TRIGGER ON UPDATE; + +-- Change an existing schedule +ALTER MATERIALIZED VIEW catalog.schema.my_mv ALTER SCHEDULE EVERY 2 HOURS; + +-- Remove a schedule +ALTER MATERIALIZED VIEW catalog.schema.my_mv DROP SCHEDULE; +``` + +### Incremental vs Full Refresh + +| Aspect | Incremental Refresh | Full Refresh | +|--------|-------------------|--------------| +| What it does | Evaluates changes since last refresh, merges only new/modified records | Re-executes the entire defining query | +| When used | When source tables support change tracking and query structure allows it | When incremental is not possible or not cost-effective | +| Requirements | Delta source tables with row tracking and CDF enabled | No special requirements | +| Cost | Lower (processes only deltas) | Higher (recomputes everything) | + +Enable row tracking on source tables for incremental refresh: + +```sql +ALTER TABLE catalog.schema.source_table +SET TBLPROPERTIES (delta.enableRowTracking = true); +``` + +By default, Databricks uses a cost model to choose between incremental and full refresh. Use `EXPLAIN CREATE MATERIALIZED VIEW` to verify the chosen refresh type. + +### Timeout Configuration + +```sql +-- Set timeout before creating or refreshing +SET STATEMENT_TIMEOUT = '6h'; +CREATE OR REFRESH MATERIALIZED VIEW catalog.schema.my_mv + SCHEDULE EVERY 12 HOURS +AS SELECT * FROM catalog.schema.large_source_table; +``` + +Default timeout is **2 days** if no warehouse timeout is configured. After changing warehouse timeouts, re-run `CREATE OR REFRESH` to apply new settings. + +### Monitoring + +- **Catalog Explorer**: View refresh status, schema, permissions, lineage under the MV entry +- **DESCRIBE EXTENDED**: Get schedule and configuration details +- **Jobs & Pipelines UI**: Monitor the automatically created pipeline +- **Pipelines API**: `GET /api/2.0/pipelines/{pipeline_id}` for programmatic access +- **DESCRIBE EXTENDED AS JSON**: Get refresh information including last refresh time, type, status, and schedule (added October 2025) + +### Key Limitations + +- No identity columns or surrogate keys +- Cannot read change data feeds (CDF) from materialized views +- Time travel queries are not supported +- `OPTIMIZE` and `VACUUM` commands are not supported (managed automatically) +- **Null handling edge case**: `SUM()` on a nullable column returns **0** instead of `NULL` when all non-null values are removed +- Non-column expressions in the defining query require explicit aliases +- Underlying storage may contain upstream data not visible in the MV definition (required for incremental refresh) +- Cannot rename the MV or change its owner via ALTER (must drop and recreate) +- No data quality expectations support +- AWS PrivateLink requires contacting Databricks support + +### DBSQL Materialized Views vs Pipeline (SDP/DLT) Materialized Views + +| Aspect | DBSQL Materialized Views | Pipeline (SDP/DLT) Materialized Views | +|--------|-------------------------|--------------------------------------| +| **Creation** | `CREATE MATERIALIZED VIEW` in SQL warehouse | Defined in pipeline source code (SQL or Python) | +| **Pipeline type** | `MV/ST` (auto-created serverless pipeline) | `ETL` (explicitly defined pipeline) | +| **Pipeline management** | Automatically created and managed | User-defined, full pipeline lifecycle control | +| **Syntax** | Standard `CREATE MATERIALIZED VIEW` | `CREATE OR REFRESH MATERIALIZED VIEW` with `PRIVATE` option | +| **Private MVs** | Not supported | `PRIVATE` keyword for pipeline-scoped views | +| **Refresh trigger** | Schedule, trigger-on-update, manual, or job-based | Pipeline update (manual or scheduled) | +| **Compute** | Serverless SQL warehouse (creation); serverless pipeline (refresh) | Pipeline compute (serverless or classic) | +| **Data quality** | Not supported | Expectations supported | +| **Best for** | Standalone MVs, BI dashboard acceleration, simple ETL | Complex multi-table pipelines, orchestrated transformations | + +Both approaches ultimately use similar underlying mechanisms (serverless pipelines) and support incremental refresh. The key difference is in management: DBSQL MVs are self-contained with auto-managed pipelines, while pipeline MVs are part of a broader orchestrated data flow. + +### Best Practices + +1. **Choose the right refresh strategy**: `TRIGGER ON UPDATE` for near-real-time SLA; `SCHEDULE` for predictable cadences; manual or job-based for complex orchestration +2. **Enable row tracking** on Delta source tables for cost-effective incremental refreshes +3. **Use async refreshes** when refresh duration is long and downstream queries can tolerate slight staleness +4. **Set explicit timeouts** when refresh duration is predictable to avoid runaway costs +5. **Use `CLUSTER BY AUTO`** for automatic liquid clustering optimization +6. **Apply row filters and column masks** at MV creation for security +7. **Monitor refresh types** with `EXPLAIN CREATE MATERIALIZED VIEW` to verify incremental behavior + +--- + +## 2. Temporary Tables and Temporary Views + +### Temporary Tables + +Temporary tables are session-scoped, physical Delta tables for intermediate data storage. They exist only within the session where they are created. + +#### Key Characteristics + +- **Session-scoped**: Only visible to the creating session; isolated from other users +- **Physical storage**: Stored as Delta tables in an internal Unity Catalog location tied to the workspace +- **Maximum lifetime**: 7 days from session creation, or until the session ends (whichever comes first) +- **No catalog privileges needed**: Any user can create temporary tables without `CREATE TABLE` privileges +- **Automatic cleanup**: Databricks reclaims storage automatically, even after unexpected disconnections +- **Shared namespace**: Temporary tables share a namespace with temporary views; you cannot create both with the same name + +#### Syntax + +```sql +-- Create with schema +CREATE TEMPORARY TABLE temp_results ( + id INT, + name STRING, + score DOUBLE +); + +-- Create from query (CTAS) +CREATE TEMP TABLE temp_active_users +AS SELECT user_id, username, last_login +FROM catalog.schema.users +WHERE last_login > current_date() - INTERVAL 30 DAYS; +``` + +Note: `CREATE OR REPLACE TEMP TABLE` is **not yet supported**. To replace, drop first. + +#### Supported Operations + +```sql +-- INSERT +INSERT INTO temp_results VALUES (1, 'Alice', 95.5); +INSERT INTO temp_results SELECT * FROM catalog.schema.source WHERE score > 90; + +-- UPDATE +UPDATE temp_results SET score = 100.0 WHERE name = 'Alice'; + +-- MERGE +MERGE INTO temp_results t +USING catalog.schema.new_scores s ON t.id = s.id +WHEN MATCHED THEN UPDATE SET score = s.score +WHEN NOT MATCHED THEN INSERT *; +``` + +#### Unsupported Operations + +- `DELETE FROM` (not supported) +- `ALTER TABLE` (drop and recreate instead) +- Shallow or deep cloning +- Time travel +- Streaming (foreachBatch) +- DataFrame API access (SQL only) + +#### Use Cases + +1. **Exploratory analysis**: Store intermediate results while iterating on queries +2. **Multi-step transformations**: Break complex transformations into readable steps +3. **Query result reuse**: Compute once, reference multiple times in a session +4. **Sandboxing**: Test transformations without affecting production tables + +#### Name Resolution + +When referencing a single-part table name, Databricks resolves in order: +1. Temporary tables in the current session +2. Permanent tables in the current schema + +Temporary tables with the same name as permanent tables **take precedence** within that session. + +### Temporary Views + +Temporary views are session-scoped, logical views that store a query definition (not data). They are recomputed on each access. + +#### Syntax + +```sql +-- Create a temporary view +CREATE TEMPORARY VIEW active_customers +AS SELECT customer_id, name, email +FROM catalog.schema.customers +WHERE status = 'active'; + +-- Replace an existing temporary view +CREATE OR REPLACE TEMPORARY VIEW active_customers +AS SELECT customer_id, name, email, phone +FROM catalog.schema.customers +WHERE status = 'active' AND last_order > current_date() - INTERVAL 90 DAYS; +``` + +#### Key Rules + +- Temporary view names **must not be qualified** (no catalog or schema prefix) +- No special privileges required to create +- Dropped automatically when the session ends +- Cannot use `schema_binding` clauses +- Support `COMMENT` and column comments + +#### Global Temporary Views (Databricks Runtime Only) + +```sql +-- Only available in Databricks Runtime, NOT in Databricks SQL +CREATE GLOBAL TEMPORARY VIEW global_summary +AS SELECT region, SUM(revenue) AS total_revenue +FROM catalog.schema.sales +GROUP BY region; + +-- Must reference via global_temp schema +SELECT * FROM global_temp.global_summary; +``` + +Global temporary views are stored in a system `global_temp` schema and are session-scoped. They are **not available in Databricks SQL** (only Databricks Runtime). + +### Temporary Tables vs Temporary Views + +| Aspect | Temporary Tables | Temporary Views | +|--------|-----------------|-----------------| +| **Storage** | Physical Delta table (stores data) | Logical (stores query definition only) | +| **Compute on access** | No (data already materialized) | Yes (query re-executed each time) | +| **DML support** | INSERT, UPDATE, MERGE | None (read-only definition) | +| **Max lifetime** | 7 days or session end | Session end | +| **CREATE OR REPLACE** | Not supported | Supported | +| **Performance** | Faster for repeated reads (data cached) | Slower for repeated reads (recomputed) | +| **Storage cost** | Uses cloud storage (auto-cleaned) | No storage cost | +| **Shared namespace** | Yes (conflicts with temp views) | Yes (conflicts with temp tables) | +| **When to use** | Large intermediate results, repeated access, DML needed | Simple query aliases, lightweight transformations | + +### Temporary Metric Views (Added September 2025) + +```sql +-- Temporary metric views: session-scoped, dropped on session end +CREATE TEMPORARY METRIC VIEW session_metrics +AS SELECT ...; +``` + +Available in Databricks Runtime 17.2+ and Databricks SQL. + +--- + +## 3. SQL Pipe Syntax + +### Overview + +Pipe syntax (introduced February 2025) allows composing SQL queries as a top-down, left-to-right chain of operations using the `|>` operator. It eliminates deeply nested subqueries and makes SQL read like a DataFrame pipeline. + +**Requirements**: Databricks SQL or Databricks Runtime **16.2+** + +### Basic Syntax + +```sql +FROM table_name +|> pipe_operation_1 +|> pipe_operation_2 +|> pipe_operation_3; +``` + +Any query can start a pipeline. The most common pattern is `FROM table_name`, but any SELECT or subquery also works: + +```sql +-- Start from a table +FROM catalog.schema.sales |> WHERE region = 'US' |> SELECT product, amount; + +-- Start from a subquery +(SELECT * FROM catalog.schema.sales WHERE year = 2025) +|> AGGREGATE SUM(amount) AS total GROUP BY product +|> ORDER BY total DESC; +``` + +### All Available Pipe Operators + +#### SELECT -- Project columns + +```sql +FROM catalog.schema.employees +|> SELECT employee_id, name, department, salary; +``` + +Note: `SELECT` in pipe syntax **must not contain aggregate functions**. Use `AGGREGATE` instead. + +#### EXTEND -- Add new columns + +Appends new columns to the existing result set (like PySpark's `withColumn`): + +```sql +FROM catalog.schema.orders +|> EXTEND quantity * unit_price AS line_total +|> EXTEND line_total * 0.1 AS tax; +``` + +Expressions can reference columns created by preceding expressions in the same EXTEND. + +#### SET -- Modify existing columns + +Overrides existing column values (like PySpark's `withColumn` on existing columns): + +```sql +FROM catalog.schema.products +|> SET price = price * 1.1 +|> SET name = UPPER(name); +``` + +Raises `UNRESOLVED_COLUMN` if the column does not exist. + +#### DROP -- Remove columns + +Removes columns (shorthand for `SELECT * EXCEPT`): + +```sql +FROM catalog.schema.users +|> DROP password_hash, internal_id, debug_flag; +``` + +#### WHERE -- Filter rows + +```sql +FROM catalog.schema.transactions +|> WHERE amount > 1000 +|> WHERE transaction_date >= '2025-01-01'; +``` + +#### AGGREGATE -- Aggregation with optional GROUP BY + +```sql +-- Full-table aggregation +FROM catalog.schema.orders +|> AGGREGATE + COUNT(*) AS total_orders, + SUM(amount) AS total_revenue, + AVG(amount) AS avg_order_value; + +-- Grouped aggregation +FROM catalog.schema.orders +|> AGGREGATE + SUM(amount) AS total_revenue, + COUNT(*) AS order_count + GROUP BY region, product_category; +``` + +In pipe syntax, `AGGREGATE` replaces `SELECT ... GROUP BY`. Numeric values in GROUP BY reference input columns, not generated results. + +#### JOIN -- Combine relations + +```sql +FROM catalog.schema.orders +|> AS o +|> LEFT JOIN catalog.schema.customers c ON o.customer_id = c.customer_id +|> SELECT o.order_id, c.name, o.amount; +``` + +All JOIN types are supported: `INNER JOIN`, `LEFT OUTER JOIN`, `RIGHT OUTER JOIN`, `FULL OUTER JOIN`, `CROSS JOIN`, `SEMI JOIN`, `ANTI JOIN`. + +#### ORDER BY -- Sort results + +```sql +FROM catalog.schema.products +|> ORDER BY price DESC, name ASC; +``` + +#### LIMIT and OFFSET -- Pagination + +```sql +FROM catalog.schema.products +|> ORDER BY price DESC +|> LIMIT 10 +|> OFFSET 20; +``` + +#### AS -- Assign table alias + +Names the intermediate result for use in subsequent JOINs or self-references: + +```sql +FROM catalog.schema.sales +|> AS current_sales +|> JOIN catalog.schema.targets t ON current_sales.region = t.region +|> SELECT current_sales.region, current_sales.revenue, t.target; +``` + +#### Set Operators -- UNION, EXCEPT, INTERSECT + +```sql +FROM catalog.schema.us_customers +|> UNION ALL (SELECT * FROM catalog.schema.eu_customers) +|> ORDER BY name; +``` + +#### TABLESAMPLE -- Sample rows + +```sql +-- Sample by row count +FROM catalog.schema.large_table +|> TABLESAMPLE (1000 ROWS); + +-- Sample by percentage +FROM catalog.schema.large_table +|> TABLESAMPLE (10 PERCENT); +``` + +#### PIVOT -- Rows to columns + +```sql +FROM catalog.schema.quarterly_sales +|> PIVOT ( + SUM(revenue) + FOR quarter IN ('Q1', 'Q2', 'Q3', 'Q4') + ); +``` + +#### UNPIVOT -- Columns to rows + +```sql +FROM catalog.schema.wide_metrics +|> UNPIVOT ( + metric_value FOR metric_name IN (cpu_usage, memory_usage, disk_usage) + ); +``` + +### Practical Examples + +#### Example 1: Multi-step aggregation (replaces nested subqueries) + +Traditional SQL: +```sql +SELECT c_count, COUNT(*) AS custdist +FROM ( + SELECT c_custkey, COUNT(o_orderkey) AS c_count + FROM customer + LEFT OUTER JOIN orders ON c_custkey = o_custkey + AND o_comment NOT LIKE '%unusual%packages%' + GROUP BY c_custkey +) AS c_orders +GROUP BY c_count +ORDER BY custdist DESC, c_count DESC; +``` + +Pipe syntax: +```sql +FROM customer +|> LEFT OUTER JOIN orders ON c_custkey = o_custkey + AND o_comment NOT LIKE '%unusual%packages%' +|> AGGREGATE COUNT(o_orderkey) AS c_count GROUP BY c_custkey +|> AGGREGATE COUNT(*) AS custdist GROUP BY c_count +|> ORDER BY custdist DESC, c_count DESC; +``` + +#### Example 2: Data exploration and profiling + +```sql +FROM catalog.schema.raw_events +|> WHERE event_date >= '2025-01-01' +|> EXTEND YEAR(event_date) AS event_year, MONTH(event_date) AS event_month +|> AGGREGATE + COUNT(*) AS event_count, + COUNT(DISTINCT user_id) AS unique_users, + AVG(duration_seconds) AS avg_duration + GROUP BY event_year, event_month +|> ORDER BY event_year, event_month; +``` + +#### Example 3: Building a report step-by-step + +```sql +FROM catalog.schema.orders +|> AS o +|> JOIN catalog.schema.products p ON o.product_id = p.product_id +|> JOIN catalog.schema.customers c ON o.customer_id = c.customer_id +|> WHERE o.order_date >= '2025-01-01' +|> EXTEND o.quantity * p.unit_price AS line_total +|> AGGREGATE + SUM(line_total) AS total_revenue, + COUNT(DISTINCT o.order_id) AS order_count + GROUP BY c.region, p.category +|> ORDER BY total_revenue DESC +|> LIMIT 20; +``` + +#### Example 4: Debugging by commenting out tail operations + +```sql +FROM catalog.schema.sales +|> WHERE region = 'US' +|> EXTEND amount * tax_rate AS tax_amount +-- |> AGGREGATE SUM(tax_amount) AS total_tax GROUP BY state +-- |> ORDER BY total_tax DESC +; +-- Comment out the last operations to inspect intermediate results +``` + +### Pipe Syntax vs Traditional SQL + +| Aspect | Traditional SQL | Pipe SQL | +|--------|----------------|----------| +| **Reading order** | Inside-out (subqueries first) | Top-down, left-to-right | +| **Clause order** | Fixed: SELECT...FROM...WHERE...GROUP BY...ORDER BY | Any order, any number of times | +| **Subquery nesting** | Required for multi-step aggregations | Eliminated via chaining | +| **Column addition** | SELECT *, expr AS new_col | `EXTEND expr AS new_col` | +| **Column removal** | SELECT with explicit column list or EXCEPT | `DROP col1, col2` | +| **Column modification** | SELECT with expression replacing column | `SET col = new_expr` | +| **Aggregation** | SELECT agg() ... GROUP BY | `AGGREGATE agg() GROUP BY` | +| **Composability** | Limited; requires CTEs or subqueries | Native chaining | +| **Interoperability** | Standard | Fully interoperable with traditional SQL | + +### When to Use Pipe Syntax + +**Use pipe syntax when:** +- Multi-step aggregations would require nested subqueries +- You want DataFrame-like readability in SQL +- Building exploratory or iterative queries (easy to add/remove steps) +- Complex transformations with many joins, filters, and projections + +**Use traditional SQL when:** +- Simple queries that are already readable +- Team is more familiar with standard SQL +- Queries will be shared with tools that may not support pipe syntax + +### Performance Considerations + +- Pipe syntax is **syntactic sugar** -- it compiles to the same execution plan as traditional SQL +- No performance difference between pipe and traditional syntax for equivalent queries +- Best practice: Place data-reducing operations (`WHERE`, `DROP`, `SELECT`) early in the pipeline to minimize data flowing through subsequent operations +- Use `TABLESAMPLE` during development to work with smaller datasets diff --git a/.claude/skills/databricks-dbsql/sql-scripting.md b/.claude/skills/databricks-dbsql/sql-scripting.md new file mode 100644 index 00000000..549a4270 --- /dev/null +++ b/.claude/skills/databricks-dbsql/sql-scripting.md @@ -0,0 +1,1077 @@ +# SQL Scripting, Stored Procedures, Recursive CTEs, and Transactions + +> Databricks SQL procedural extensions based on the SQL/PSM standard. Covers SQL scripting (compound statements, control flow, exception handling), stored procedures, recursive CTEs, and multi-statement transactions. + +--- + +## Table of Contents + +- [SQL Scripting](#sql-scripting) + - [Compound Statements (BEGIN...END)](#compound-statements-beginend) + - [Variable Declaration (DECLARE)](#variable-declaration-declare) + - [Variable Assignment (SET)](#variable-assignment-set) + - [Control Flow](#control-flow) + - [IF / ELSEIF / ELSE](#if--elseif--else) + - [CASE Statement](#case-statement) + - [WHILE Loop](#while-loop) + - [FOR Loop](#for-loop) + - [LOOP Statement](#loop-statement) + - [REPEAT Statement](#repeat-statement) + - [LEAVE and ITERATE](#leave-and-iterate) + - [Exception Handling](#exception-handling) + - [Condition Declaration](#condition-declaration) + - [Handler Declaration](#handler-declaration) + - [SIGNAL and RESIGNAL](#signal-and-resignal) + - [EXECUTE IMMEDIATE (Dynamic SQL)](#execute-immediate-dynamic-sql) +- [Stored Procedures](#stored-procedures) + - [CREATE PROCEDURE](#create-procedure) + - [CALL (Invoke a Procedure)](#call-invoke-a-procedure) + - [DROP PROCEDURE](#drop-procedure) + - [DESCRIBE PROCEDURE](#describe-procedure) + - [SHOW PROCEDURES](#show-procedures) +- [Recursive CTEs](#recursive-ctes) + - [WITH RECURSIVE Syntax](#with-recursive-syntax) + - [Anchor and Recursive Members](#anchor-and-recursive-members) + - [MAX RECURSION LEVEL](#max-recursion-level) + - [Use Cases and Examples](#use-cases-and-examples) + - [Limitations](#limitations) +- [Multi-Statement Transactions](#multi-statement-transactions) + - [Overview and Current Status](#overview-and-current-status) + - [SQL Scripting Atomic Blocks](#sql-scripting-atomic-blocks) + - [Python Connector Transaction API](#python-connector-transaction-api) + - [Isolation Levels](#isolation-levels) + - [Write Conflicts and Concurrency](#write-conflicts-and-concurrency) + - [Best Practices](#best-practices) + +--- + +## SQL Scripting + +**Availability**: Databricks Runtime 16.3+ and Databricks SQL + +SQL scripting enables procedural logic using the SQL/PSM standard. Every SQL script starts with a compound statement block (`BEGIN...END`). + +### Compound Statements (BEGIN...END) + +A compound statement is the fundamental building block containing variable declarations, condition/handler declarations, and executable statements. + +**Syntax**: + +```sql +[ label : ] BEGIN + [ { declare_variable | declare_condition } ; [...] ] + [ declare_handler ; [...] ] + [ SQL_statement ; [...] ] +END [ label ] +``` + +**Key rules**: + +- Declarations must appear before executable statements +- Variable declarations come before condition declarations, which come before handler declarations +- Top-level compound statements cannot specify labels +- `NOT ATOMIC` is the default and only behavior (no automatic rollback on failure) +- In notebooks, the compound statement must be the sole statement in the cell + +**Supported statement types in body**: + +| Category | Statements | +|----------|-----------| +| DDL | ALTER, CREATE, DROP | +| DCL | GRANT, REVOKE | +| DML | INSERT, UPDATE, DELETE, MERGE | +| Query | SELECT | +| Assignment | SET | +| Dynamic SQL | EXECUTE IMMEDIATE | +| Control flow | IF, CASE, WHILE, FOR, LOOP, REPEAT, LEAVE, ITERATE | +| Nesting | Nested BEGIN...END blocks | + +**Minimal example**: + +```sql +BEGIN + SELECT 'Hello, SQL Scripting!'; +END; +``` + +### Variable Declaration (DECLARE) + +**Syntax**: + +```sql +DECLARE variable_name [, ...] data_type [ DEFAULT default_expr ]; +``` + +- Variables initialize to `NULL` if no `DEFAULT` is specified +- Data type can be omitted when `DEFAULT` is provided (type inferred from expression) +- Multiple variable names in a single `DECLARE` supported in Runtime 17.2+ +- Variables are scoped to their enclosing compound statement +- Variable names resolve from the innermost scope outward; use labels to disambiguate + +**Examples**: + +```sql +BEGIN + DECLARE counter INT DEFAULT 0; + DECLARE name STRING DEFAULT 'unknown'; + DECLARE x, y, z DOUBLE DEFAULT 0.0; -- Runtime 17.2+ + DECLARE inferred DEFAULT current_date(); -- type inferred as DATE + + SET counter = counter + 1; + VALUES (counter, name); +END; +``` + +### Variable Assignment (SET) + +**Syntax**: + +```sql +SET variable_name = expression; +SET VAR variable_name = expression; -- explicit local variable +SET (var1, var2, ...) = (expr1, expr2, ...); -- multi-assignment +``` + +Use `SET VAR` to explicitly target a local variable when a session variable with the same name exists. + +**Example**: + +```sql +BEGIN + DECLARE total INT DEFAULT 0; + DECLARE label STRING; + SET total = 100; + SET label = 'final'; + VALUES (total, label); +END; +``` + +### Control Flow + +#### IF / ELSEIF / ELSE + +Executes statements based on the first condition evaluating to `TRUE`. + +**Syntax**: + +```sql +IF condition THEN + { stmt ; } [...] +[ ELSEIF condition THEN + { stmt ; } [...] ] [...] +[ ELSE + { stmt ; } [...] ] +END IF; +``` + +**Example**: + +```sql +BEGIN + DECLARE score INT DEFAULT 85; + DECLARE grade STRING; + + IF score >= 90 THEN + SET grade = 'A'; + ELSEIF score >= 80 THEN + SET grade = 'B'; + ELSEIF score >= 70 THEN + SET grade = 'C'; + ELSE + SET grade = 'F'; + END IF; + + VALUES (grade); -- Returns 'B' +END; +``` + +#### CASE Statement + +Two forms: **simple CASE** (compare expression) and **searched CASE** (evaluate boolean conditions). + +**Simple CASE syntax**: + +```sql +CASE expr + WHEN opt1 THEN { stmt ; } [...] + WHEN opt2 THEN { stmt ; } [...] + [ ELSE { stmt ; } [...] ] +END CASE; +``` + +**Searched CASE syntax**: + +```sql +CASE + WHEN cond1 THEN { stmt ; } [...] + WHEN cond2 THEN { stmt ; } [...] + [ ELSE { stmt ; } [...] ] +END CASE; +``` + +Only the first matching branch executes. + +**Example**: + +```sql +BEGIN + DECLARE status STRING DEFAULT 'active'; + + CASE status + WHEN 'active' THEN VALUES ('Processing'); + WHEN 'paused' THEN VALUES ('On hold'); + WHEN 'archived' THEN VALUES ('Read-only'); + ELSE VALUES ('Unknown status'); + END CASE; +END; +``` + +#### WHILE Loop + +Repeats while a condition is `TRUE`. + +**Syntax**: + +```sql +[ label : ] WHILE condition DO + { stmt ; } [...] +END WHILE [ label ]; +``` + +**Example** -- sum odd numbers from 1 to 10: + +```sql +BEGIN + DECLARE total INT DEFAULT 0; + DECLARE i INT DEFAULT 0; + + sum_odds: WHILE i < 10 DO + SET i = i + 1; + IF i % 2 = 0 THEN + ITERATE sum_odds; -- skip even numbers + END IF; + SET total = total + i; + END WHILE sum_odds; + + VALUES (total); -- Returns 25 +END; +``` + +#### FOR Loop + +Iterates over query result rows. + +**Syntax**: + +```sql +[ label : ] FOR [ variable_name AS ] query DO + { stmt ; } [...] +END FOR [ label ]; +``` + +- Use `variable_name` (not the label) to qualify column references from the cursor +- For Delta tables, modifying the source during iteration does not affect cursor results +- Loop may not fully execute the query if terminated early by `LEAVE` or an error + +**Example** -- process each row from a query: + +```sql +BEGIN + DECLARE total_revenue DOUBLE DEFAULT 0.0; + + process_orders: FOR row AS + SELECT order_id, amount FROM orders WHERE status = 'completed' + DO + SET total_revenue = total_revenue + row.amount; + IF total_revenue > 1000000 THEN + LEAVE process_orders; + END IF; + END FOR process_orders; + + VALUES (total_revenue); +END; +``` + +#### LOOP Statement + +Unconditional loop; must use `LEAVE` to exit. + +**Syntax**: + +```sql +[ label : ] LOOP + { stmt ; } [...] +END LOOP [ label ]; +``` + +**Example**: + +```sql +BEGIN + DECLARE counter INT DEFAULT 0; + + count_up: LOOP + SET counter = counter + 1; + IF counter >= 5 THEN + LEAVE count_up; + END IF; + END LOOP count_up; + + VALUES (counter); -- Returns 5 +END; +``` + +#### REPEAT Statement + +Executes at least once, then repeats until condition is `TRUE`. + +**Syntax**: + +```sql +[ label : ] REPEAT + { stmt ; } [...] + UNTIL condition +END REPEAT [ label ]; +``` + +**Example**: + +```sql +BEGIN + DECLARE total INT DEFAULT 0; + DECLARE i INT DEFAULT 0; + + sum_loop: REPEAT + SET i = i + 1; + IF i % 2 != 0 THEN + SET total = total + i; + END IF; + UNTIL i >= 10 + END REPEAT sum_loop; + + VALUES (total); -- Returns 25 +END; +``` + +#### LEAVE and ITERATE + +| Statement | Purpose | Equivalent | +|-----------|---------|-----------| +| `LEAVE label` | Exit the labeled loop or compound block | `BREAK` in other languages | +| `ITERATE label` | Skip to the next iteration of the labeled loop | `CONTINUE` in other languages | + +Both require a labeled loop to target. + +### Exception Handling + +#### Condition Declaration + +Define named conditions for specific SQLSTATE codes. + +**Syntax**: + +```sql +DECLARE condition_name CONDITION [ FOR SQLSTATE [ VALUE ] sqlstate ]; +``` + +- `sqlstate` is a 5-character alphanumeric string (A-Z, 0-9, case-insensitive) +- Cannot start with `'00'`, `'01'`, or `'XX'` +- Defaults to `'45000'` if not specified + +**Example**: + +```sql +BEGIN + DECLARE divide_by_zero CONDITION FOR SQLSTATE '22012'; + -- Use in handler declarations below +END; +``` + +#### Handler Declaration + +Catch and handle exceptions within compound statements. + +**Syntax**: + +```sql +DECLARE handler_type HANDLER FOR condition_value [, ...] handler_action; +``` + +| Parameter | Options | Description | +|-----------|---------|-------------| +| `handler_type` | `EXIT` | Exits the enclosing compound after handling | +| `condition_value` | `SQLSTATE 'xxxxx'`, `condition_name`, `SQLEXCEPTION`, `NOT FOUND` | What to catch | +| `handler_action` | Single statement or nested `BEGIN...END` | What to execute | + +- `SQLEXCEPTION` catches all error states (SQLSTATE class not `'00'` or `'01'`) +- `NOT FOUND` catches `'02xxx'` states (no data found) +- A handler cannot apply to statements in its own body + +**Example** -- catch division by zero: + +```sql +BEGIN + DECLARE result DOUBLE; + DECLARE EXIT HANDLER FOR SQLSTATE '22012' + BEGIN + SET result = -1; + END; + + SET result = 10 / 0; -- triggers handler + VALUES (result); -- Returns -1 +END; +``` + +**Example** -- generic exception handler: + +```sql +BEGIN + DECLARE error_msg STRING DEFAULT 'none'; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + SET error_msg = 'An error occurred'; + INSERT INTO error_log (message, ts) VALUES (error_msg, current_timestamp()); + END; + + -- statements that might fail + INSERT INTO target_table SELECT * FROM source_table; +END; +``` + +#### SIGNAL and RESIGNAL + +Raise or re-raise exceptions. + +**SIGNAL syntax**: + +```sql +SIGNAL condition_name + [ SET { MESSAGE_ARGUMENTS = argument_map | MESSAGE_TEXT = message_str } ]; + +SIGNAL SQLSTATE [ VALUE ] sqlstate + [ SET MESSAGE_TEXT = message_str ]; +``` + +**RESIGNAL syntax** (use in handlers to preserve diagnostic stack): + +```sql +RESIGNAL [ condition_name | SQLSTATE [ VALUE ] sqlstate ] + [ SET { MESSAGE_ARGUMENTS = argument_map | MESSAGE_TEXT = message_str } ]; +``` + +- Prefer `RESIGNAL` over `SIGNAL` inside handlers -- `RESIGNAL` preserves the diagnostic stack while `SIGNAL` clears it +- `MESSAGE_ARGUMENTS` takes a `MAP` literal + +**Example** -- validate input and raise custom error: + +```sql +BEGIN + DECLARE input_value INT DEFAULT 150; + + IF input_value > 100 THEN + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = 'Input value must be <= 100'; + END IF; + + VALUES (input_value); +END; +``` + +**Example** -- using named conditions with MESSAGE_ARGUMENTS: + +```sql +BEGIN + DECLARE input INT DEFAULT 5; + DECLARE arg_map MAP; + + IF input > 4 THEN + SET arg_map = map('errorMessage', 'Input must be <= 4.'); + SIGNAL USER_RAISED_EXCEPTION + SET MESSAGE_ARGUMENTS = arg_map; + END IF; +END; +``` + +### EXECUTE IMMEDIATE (Dynamic SQL) + +Execute SQL statements constructed as strings at runtime. + +**Availability**: Runtime 14.3+; expression-based `sql_string` and nested execution from Runtime 17.3+. + +**Syntax**: + +```sql +EXECUTE IMMEDIATE sql_string + [ INTO var_name [, ...] ] + [ USING { arg_expr [ AS ] [ alias ] } [, ...] ]; +``` + +- `sql_string`: a constant expression producing a well-formed SQL statement +- `INTO`: captures a single-row result into variables (returns `NULL` for zero rows; errors for multiple rows) +- `USING`: binds values to positional (`?`) or named (`:param`) parameter markers (cannot mix styles) + +**Examples**: + +```sql +-- Positional parameters +EXECUTE IMMEDIATE 'SELECT SUM(c1) FROM VALUES(?), (?) AS t(c1)' USING 5, 6; + +-- Named parameters with INTO +BEGIN + DECLARE total INT; + EXECUTE IMMEDIATE 'SELECT SUM(c1) FROM VALUES(:a), (:b) AS t(c1)' + INTO total USING (5 AS a, 6 AS b); + VALUES (total); -- Returns 11 +END; + +-- Dynamic table operations +BEGIN + DECLARE table_name STRING DEFAULT 'my_catalog.my_schema.staging'; + EXECUTE IMMEDIATE 'TRUNCATE TABLE ' || table_name; + EXECUTE IMMEDIATE 'INSERT INTO ' || table_name || ' SELECT * FROM source'; +END; +``` + +--- + +## Stored Procedures + +**Availability**: Public Preview -- Databricks Runtime 17.0+ + +Stored procedures persist SQL scripts in Unity Catalog and are invoked with `CALL`. + +### CREATE PROCEDURE + +**Syntax**: + +```sql +CREATE [ OR REPLACE ] PROCEDURE [ IF NOT EXISTS ] + procedure_name ( [ parameter [, ...] ] ) + characteristic [...] + AS compound_statement +``` + +**Parameter definition**: + +```sql +[ IN | OUT | INOUT ] parameter_name data_type + [ DEFAULT default_expression ] + [ COMMENT parameter_comment ] +``` + +| Parameter mode | Behavior | +|---------------|----------| +| `IN` (default) | Input-only; value passed into the procedure | +| `OUT` | Output-only; initialized to `NULL`; final value returned on success | +| `INOUT` | Input and output; accepts a value and returns the modified value on success | + +**Required characteristics**: + +| Characteristic | Description | +|---------------|-------------| +| `LANGUAGE SQL` | Specifies the implementation language | +| `SQL SECURITY INVOKER` | Executes under the invoker's authority | + +**Optional characteristics**: + +| Characteristic | Description | +|---------------|-------------| +| `NOT DETERMINISTIC` | Procedure may return different results with identical inputs | +| `MODIFIES SQL DATA` | Procedure modifies SQL data | +| `COMMENT 'description'` | Human-readable description | +| `DEFAULT COLLATION UTF8_BINARY` | Required when schema uses non-UTF8_BINARY collation (Runtime 17.1+) | + +**Rules**: + +- `OR REPLACE` and `IF NOT EXISTS` cannot be combined +- Parameter names must be unique within the procedure +- `DEFAULT` is not supported for `OUT` parameters +- Once a parameter has a `DEFAULT`, all subsequent parameters must also have defaults +- Default expressions cannot reference other parameters or contain subqueries +- Body is validated syntactically at creation but semantically only at invocation + +**Example** -- ETL procedure with output parameters: + +```sql +CREATE OR REPLACE PROCEDURE run_daily_etl( + IN source_schema STRING, + IN target_schema STRING, + OUT rows_processed INT, + OUT status STRING DEFAULT 'pending' +) +LANGUAGE SQL +SQL SECURITY INVOKER +COMMENT 'Daily ETL pipeline for order processing' +AS BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + SET status = 'failed'; + SET rows_processed = 0; + END; + + -- Truncate and reload + EXECUTE IMMEDIATE 'TRUNCATE TABLE ' || target_schema || '.orders_daily'; + + EXECUTE IMMEDIATE + 'INSERT INTO ' || target_schema || '.orders_daily ' + || 'SELECT * FROM ' || source_schema || '.orders ' + || 'WHERE order_date = current_date()'; + + EXECUTE IMMEDIATE + 'SELECT COUNT(*) FROM ' || target_schema || '.orders_daily' + INTO rows_processed; + + SET status = 'success'; +END; +``` + +### CALL (Invoke a Procedure) + +**Syntax**: + +```sql +CALL procedure_name( [ argument [, ...] ] ); +CALL procedure_name( [ named_param => argument ] [, ...] ); +``` + +**Rules**: + +- Supports up to 64 levels of nesting +- For `IN` parameters: any expression castable to the parameter type, or `DEFAULT` +- For `OUT`/`INOUT` parameters: must be a session variable or local variable +- Arguments must match the data type of the parameter (use typed literals, e.g., `DATE'2025-01-01'`) +- Fewer arguments allowed if remaining parameters have `DEFAULT` values +- Not supported via ODBC + +**Example**: + +```sql +-- Positional invocation +DECLARE rows_out INT; +DECLARE status_out STRING; +CALL run_daily_etl('raw', 'silver', rows_out, status_out); +SELECT rows_out, status_out; + +-- Named parameter invocation +CALL run_daily_etl( + target_schema => 'silver', + source_schema => 'raw', + rows_processed => rows_out, + status => status_out +); +``` + +### DROP PROCEDURE + +**Syntax**: + +```sql +DROP PROCEDURE [ IF EXISTS ] procedure_name; +``` + +- Without `IF EXISTS`, dropping a non-existent procedure raises `ROUTINE_NOT_FOUND` +- Requires `MANAGE` privilege, ownership of the procedure, or ownership of the containing schema/catalog/metastore + +**Example**: + +```sql +DROP PROCEDURE IF EXISTS run_daily_etl; +``` + +### DESCRIBE PROCEDURE + +**Syntax**: + +```sql +{ DESC | DESCRIBE } PROCEDURE [ EXTENDED ] procedure_name; +``` + +- Basic: returns procedure name and parameter list +- `EXTENDED`: additionally returns owner, creation time, body, language, security type, determinism, data access, and configuration + +**Example**: + +```sql +DESCRIBE PROCEDURE EXTENDED run_daily_etl; +``` + +### SHOW PROCEDURES + +**Syntax**: + +```sql +SHOW PROCEDURES [ { FROM | IN } schema_name ]; +``` + +Returns columns: `catalog`, `namespace`, `schema`, `procedure_name`. + +**Example**: + +```sql +SHOW PROCEDURES IN my_catalog.my_schema; +``` + +--- + +## Recursive CTEs + +**Availability**: Databricks Runtime 17.0+ and DBSQL 2025.20+ + +Recursive CTEs enable self-referential queries for hierarchical data, graph traversal, and series generation. + +### WITH RECURSIVE Syntax + +```sql +WITH RECURSIVE cte_name [ ( column_name [, ...] ) ] + [ MAX RECURSION LEVEL max_level ] AS ( + base_case_query + UNION ALL + recursive_query + ) +SELECT ... FROM cte_name; +``` + +### Anchor and Recursive Members + +| Component | Description | +|-----------|-------------| +| **Anchor (base case)** | Initial query providing seed rows; must NOT reference the CTE name | +| **Recursive member** | References the CTE name; processes rows from the previous iteration | +| **UNION ALL** | Combines anchor and recursive results (required) | + +The recursive member reads rows produced by the previous iteration and generates new rows. Recursion terminates when the recursive member produces zero rows. + +### MAX RECURSION LEVEL + +```sql +WITH RECURSIVE cte_name MAX RECURSION LEVEL 200 AS (...) +``` + +| Setting | Default | Description | +|---------|---------|-------------| +| Max recursion depth | 100 | Exceeding raises `RECURSION_LEVEL_LIMIT_EXCEEDED` | +| Max result rows | 1,000,000 | Exceeding raises an error | +| `LIMIT ALL` | N/A | Suspends the row limit (Runtime 17.2+) | + +### Use Cases and Examples + +**Generate a number series**: + +```sql +WITH RECURSIVE numbers(n) AS ( + VALUES (1) + UNION ALL + SELECT n + 1 FROM numbers WHERE n < 100 +) +SELECT * FROM numbers; +``` + +**Organizational hierarchy traversal**: + +```sql +WITH RECURSIVE org_tree AS ( + -- Anchor: start from the CEO + SELECT employee_id, name, manager_id, name AS root_name, 0 AS depth + FROM employees + WHERE manager_id IS NULL + + UNION ALL + + -- Recursive: find direct reports + SELECT e.employee_id, e.name, e.manager_id, t.root_name, t.depth + 1 + FROM employees e + JOIN org_tree t ON e.manager_id = t.employee_id +) +SELECT * FROM org_tree ORDER BY depth, name; +``` + +**Graph traversal with cycle detection**: + +```sql +WITH RECURSIVE search_graph(f, t, label, path, cycle) AS ( + -- Anchor: all edges as starting paths + SELECT *, array(struct(g.f, g.t)), false + FROM graph g + + UNION ALL + + -- Recursive: extend paths, detect cycles + SELECT g.f, g.t, g.label, + sg.path || array(struct(g.f, g.t)), + array_contains(sg.path, struct(g.f, g.t)) + FROM graph g + JOIN search_graph sg ON g.f = sg.t + WHERE NOT sg.cycle +) +SELECT * FROM search_graph WHERE NOT cycle; +``` + +**String accumulation**: + +```sql +WITH RECURSIVE r(col) AS ( + SELECT 'a' + UNION ALL + SELECT col || char(ascii(substr(col, -1)) + 1) + FROM r + WHERE length(col) < 10 +) +SELECT * FROM r; +-- a, ab, abc, abcd, ..., abcdefghij +``` + +**Bill of Materials (BOM) explosion**: + +```sql +WITH RECURSIVE bom AS ( + -- Anchor: top-level product + SELECT part_id, component_id, quantity, 1 AS level + FROM bill_of_materials + WHERE part_id = 'PROD-001' + + UNION ALL + + -- Recursive: sub-components + SELECT b.part_id, b.component_id, b.quantity * bom.quantity, bom.level + 1 + FROM bill_of_materials b + JOIN bom ON b.part_id = bom.component_id +) +SELECT component_id, SUM(quantity) AS total_quantity, MAX(level) AS max_depth +FROM bom +GROUP BY component_id +ORDER BY total_quantity DESC; +``` + +### Limitations + +- Not supported in UPDATE, DELETE, or MERGE statements +- Step (recursive) queries cannot include correlated column references to the CTE name +- Random number generators may produce identical values across iterations +- Default row limit of 1,000,000 rows (use `LIMIT ALL` in Runtime 17.2+ to override) +- Default recursion depth of 100 (override with `MAX RECURSION LEVEL`) + +--- + +## Multi-Statement Transactions + +### Overview and Current Status + +Multi-statement transactions (MST) allow grouping multiple SQL statements into atomic units that either succeed completely or fail completely. + +| Feature | Status | Notes | +|---------|--------|-------| +| Single-table transactions | GA | Delta Lake default; every DML statement is atomic | +| Multi-statement transactions (SQL scripting) | Preview | `BEGIN ATOMIC...END` blocks | +| Multi-statement transactions (Python connector) | Preview | `connection.autocommit = False` pattern | +| Cross-table transactions | Preview | Atomic updates across multiple Delta tables | + +### SQL Scripting Atomic Blocks + +Use `BEGIN ATOMIC...END` to execute multiple statements as a single atomic unit: + +```sql +BEGIN ATOMIC + INSERT INTO customers (id, name) VALUES (1, 'Alice'); + INSERT INTO orders (id, customer_id, amount) VALUES (1, 1, 250.00); + INSERT INTO audit_log (action, ts) VALUES ('new_customer_order', current_timestamp()); +END; +``` + +If any statement fails, all changes are rolled back. + +> **Note:** Tables used in `BEGIN ATOMIC` blocks must have the `catalogManaged` table feature enabled. Create tables with `TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported')`. Existing tables cannot be upgraded in place — they must be recreated with this property. + +### Python Connector Transaction API + +The Databricks SQL Connector for Python provides explicit transaction control: + +```python +from databricks import sql + +connection = sql.connect( + server_hostname="...", + http_path="...", + access_token="..." +) + +# Disable autocommit to start explicit transactions +connection.autocommit = False +cursor = connection.cursor() + +try: + cursor.execute("INSERT INTO customers VALUES (1, 'Alice')") + cursor.execute("INSERT INTO orders VALUES (1, 1, 100.00)") + cursor.execute("INSERT INTO shipments VALUES (1, 1, 'pending')") + connection.commit() # All three succeed atomically +except Exception: + connection.rollback() # All three discarded +finally: + connection.autocommit = True +``` + +**Key API methods**: + +| Method | Description | +|--------|-------------| +| `connection.autocommit = False` | Start explicit transaction mode | +| `connection.commit()` | Commit the current transaction | +| `connection.rollback()` | Discard all changes in the current transaction | +| `connection.get_transaction_isolation()` | Returns current isolation level | +| `connection.set_transaction_isolation(level)` | Sets isolation level | + +**Error handling**: + +- `sql.TransactionError` raised when committing without an active transaction +- Cannot change `autocommit` while a transaction is active +- `rollback()` is a safe no-op when no transaction is active + +### Isolation Levels + +Databricks uses **Snapshot Isolation** (mapped to `REPEATABLE_READ` in standard SQL terminology). + +| Level | Description | Default | +|-------|-------------|---------| +| `WriteSerializable` | Only writes are serializable; concurrent writes may reorder | Yes (table default) | +| `Serializable` | Both reads and writes are serializable; strictest isolation | No | +| `REPEATABLE_READ` | Snapshot isolation for connector-level transactions | Connector default | + +**Setting isolation at table level**: + +```sql +ALTER TABLE my_table +SET TBLPROPERTIES ('delta.isolationLevel' = 'Serializable'); +``` + +**Setting isolation in Python connector**: + +```python +from databricks.sql import TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ + +connection.set_transaction_isolation(TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ) +# Only REPEATABLE_READ is supported; others raise NotSupportedError +``` + +**Snapshot isolation behavior**: + +- **Repeatable reads**: Data read within a transaction remains consistent +- **Atomic commits**: Changes are invisible to other connections until committed +- **Write conflicts**: Concurrent writes to the same table cause conflicts +- **Cross-table writes**: Concurrent writes to different tables can succeed + +### Write Conflicts and Concurrency + +**Row-level concurrency** (Runtime 14.2+) reduces conflicts for tables with deletion vectors or liquid clustering: + +| Operation | WriteSerializable | Serializable | +|-----------|------------------|--------------| +| INSERT vs INSERT | No conflict | No conflict | +| UPDATE/DELETE/MERGE vs same | No conflict (different rows) | May conflict | +| OPTIMIZE vs concurrent DML | Conflict only with ZORDER BY | May conflict | + +**Common conflict exceptions**: + +| Exception | Cause | +|-----------|-------| +| `ConcurrentAppendException` | Concurrent append to the same partition | +| `ConcurrentDeleteReadException` | Concurrent delete of files being read | +| `MetadataChangedException` | Concurrent ALTER TABLE or schema change | +| `ProtocolChangedException` | Protocol version upgrade during write | + +### Best Practices + +1. **Keep transactions short** to minimize conflict windows +2. **Always wrap in try/except/finally** with rollback on errors +3. **Restore autocommit** in the `finally` block +4. **Use partition pruning** in MERGE conditions to reduce conflict scope +5. **Enable row-level concurrency** (deletion vectors + liquid clustering) for high-concurrency workloads +6. **Prefer single-statement MERGE** over multi-statement transactions when updating a single table +7. **Commit and restart** transactions to see changes made by other connections + +--- + +## Runtime Version Reference + +| Feature | Minimum Runtime | Status | +|---------|----------------|--------| +| SQL Scripting (compound statements, control flow) | 16.3 | GA | +| Stored Procedures (CREATE/CALL/DROP PROCEDURE) | 17.0 | Public Preview | +| Recursive CTEs (WITH RECURSIVE) | 17.0 / DBSQL 2025.20 | GA | +| Multi-variable DECLARE | 17.2 | GA | +| EXECUTE IMMEDIATE (basic) | 14.3 | GA | +| EXECUTE IMMEDIATE (expressions, nested) | 17.3 | GA | +| Recursive CTE LIMIT ALL | 17.2 | GA | +| Multi-statement Transactions | Varies | Preview | +| Row-level Concurrency | 14.2 | GA | + +--- + +## Quick Reference Card + +### SQL Scripting Skeleton + +```sql +BEGIN + -- 1. Declarations + DECLARE var1 INT DEFAULT 0; + DECLARE var2 STRING; + DECLARE my_error CONDITION FOR SQLSTATE '45000'; + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + -- error handling logic + END; + + -- 2. Logic + IF var1 > 0 THEN + SET var2 = 'positive'; + ELSE + SET var2 = 'non-positive'; + END IF; + + -- 3. Output + VALUES (var1, var2); +END; +``` + +### Stored Procedure Skeleton + +```sql +CREATE OR REPLACE PROCEDURE my_schema.my_proc( + IN input_param STRING, + OUT output_param INT +) +LANGUAGE SQL +SQL SECURITY INVOKER +COMMENT 'Description of what this procedure does' +AS BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + SET output_param = -1; + + -- procedure body + SET output_param = (SELECT COUNT(*) FROM my_table WHERE col = input_param); +END; + +-- Invoke +DECLARE result INT; +CALL my_schema.my_proc('value', result); +SELECT result; +``` + +### Recursive CTE Skeleton + +```sql +WITH RECURSIVE cte_name (col1, col2) MAX RECURSION LEVEL 50 AS ( + -- Anchor + SELECT seed_col1, seed_col2 + FROM base_table + WHERE condition + + UNION ALL + + -- Recursive step + SELECT derived_col1, derived_col2 + FROM source_table s + JOIN cte_name c ON s.parent = c.col1 +) +SELECT * FROM cte_name; +``` diff --git a/.claude/skills/databricks-docs/SKILL.md b/.claude/skills/databricks-docs/SKILL.md index 98aeac9b..54bb157f 100644 --- a/.claude/skills/databricks-docs/SKILL.md +++ b/.claude/skills/databricks-docs/SKILL.md @@ -16,7 +16,7 @@ This is a **reference skill**, not an action skill. Use it to: - Find detailed information to inform how you use MCP tools - Discover features and capabilities you may not know about -**Always prefer using MCP tools for actions** (execute_sql, create_or_update_pipeline, etc.) and **load specific skills for workflows** (databricks-python-sdk, spark-declarative-pipelines, etc.). Use this skill when you need reference documentation. +**Always prefer using MCP tools for actions** (execute_sql, create_or_update_pipeline, etc.) and **load specific skills for workflows** (databricks-python-sdk, databricks-spark-declarative-pipelines, etc.). Use this skill when you need reference documentation. ## How to Use @@ -45,7 +45,7 @@ The llms.txt file is organized by category: **Scenario:** User wants to create a Delta Live Tables pipeline -1. Load `spark-declarative-pipelines` skill for workflow patterns +1. Load `databricks-spark-declarative-pipelines` skill for workflow patterns 2. Use this skill to fetch docs if you need clarification on specific DLT features 3. Use `create_or_update_pipeline` MCP tool to actually create the pipeline @@ -54,3 +54,11 @@ The llms.txt file is organized by category: 1. Fetch llms.txt to find relevant documentation 2. Read the specific docs to understand the feature 3. Determine which skill/tools apply, then use them + +## Related Skills + +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - SDK patterns for programmatic Databricks access +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - DLT / Lakeflow pipeline workflows +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Governance and catalog management +- **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - Serving endpoints and model deployment +- **[databricks-mlflow-evaluation](../databricks-mlflow-evaluation/SKILL.md)** - MLflow 3 GenAI evaluation workflows diff --git a/.claude/skills/databricks-genie/SKILL.md b/.claude/skills/databricks-genie/SKILL.md index 4d5d12f5..3f08628c 100644 --- a/.claude/skills/databricks-genie/SKILL.md +++ b/.claude/skills/databricks-genie/SKILL.md @@ -109,8 +109,8 @@ Before creating a Genie Space: ### Creating Tables Use these skills in sequence: -1. `synthetic-data-generation` - Generate raw parquet files -2. `spark-declarative-pipelines` - Create bronze/silver/gold tables +1. `databricks-synthetic-data-generation` - Generate raw parquet files +2. `databricks-spark-declarative-pipelines` - Create bronze/silver/gold tables ## Common Issues @@ -119,3 +119,10 @@ Use these skills in sequence: | **No warehouse available** | Create a SQL warehouse or provide `warehouse_id` explicitly | | **Poor query generation** | Add instructions and sample questions that reference actual column names | | **Slow queries** | Ensure warehouse is running; use OPTIMIZE on tables | + +## Related Skills + +- **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Use Genie Spaces as agents inside Supervisor Agents +- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate raw parquet data to populate tables for Genie +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Build bronze/silver/gold tables consumed by Genie Spaces +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Manage the catalogs, schemas, and tables Genie queries diff --git a/.claude/skills/databricks-genie/spaces.md b/.claude/skills/databricks-genie/spaces.md index 71c93985..8549d6bd 100644 --- a/.claude/skills/databricks-genie/spaces.md +++ b/.claude/skills/databricks-genie/spaces.md @@ -163,10 +163,10 @@ The tool finds the existing space by name and updates it. ## Example End-to-End Workflow -1. **Generate synthetic data** using `synthetic-data-generation` skill: +1. **Generate synthetic data** using `databricks-synthetic-data-generation` skill: - Creates parquet files in `/Volumes/catalog/schema/raw_data/` -2. **Create tables** using `spark-declarative-pipelines` skill: +2. **Create tables** using `databricks-spark-declarative-pipelines` skill: - Creates `catalog.schema.bronze_*` → `catalog.schema.silver_*` → `catalog.schema.gold_*` 3. **Inspect the tables**: diff --git a/.claude/skills/databricks-jobs/SKILL.md b/.claude/skills/databricks-jobs/SKILL.md index eae0754e..2f0f8c73 100644 --- a/.claude/skills/databricks-jobs/SKILL.md +++ b/.claude/skills/databricks-jobs/SKILL.md @@ -326,8 +326,8 @@ resources: ## Related Skills -- **[asset-bundles](../asset-bundles/SKILL.md)** - Deploy jobs via Databricks Asset Bundles -- **[spark-declarative-pipelines](../spark-declarative-pipelines/SKILL.md)** - Configure pipelines triggered by jobs +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - Deploy jobs via Databricks Asset Bundles +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Configure pipelines triggered by jobs ## Resources diff --git a/.claude/skills/databricks-jobs/task-types.md b/.claude/skills/databricks-jobs/task-types.md index a78b9a32..c5b06fbe 100644 --- a/.claude/skills/databricks-jobs/task-types.md +++ b/.claude/skills/databricks-jobs/task-types.md @@ -618,7 +618,7 @@ Define reusable Python environments for serverless tasks with custom pip depende > **IMPORTANT:** The `client` field is **required** in the environment `spec`. It specifies the > base serverless environment version. Use `"4"` as the value. Without it, the API returns: > `"Either base environment or version must be provided for environment"`. -> The MCP `create_job` tool auto-injects `client: "4"` if omitted, but CLI/SDK calls require it explicitly. +> The MCP `manage_jobs` tool (action="create") auto-injects `client: "4"` if omitted, but CLI/SDK calls require it explicitly. ### DABs YAML diff --git a/.claude/skills/databricks-lakebase-autoscale/SKILL.md b/.claude/skills/databricks-lakebase-autoscale/SKILL.md new file mode 100644 index 00000000..50ba1df7 --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/SKILL.md @@ -0,0 +1,294 @@ +--- +name: databricks-lakebase-autoscale +description: "Patterns and best practices for using Lakebase Autoscaling (next-gen managed PostgreSQL) with autoscaling, branching, scale-to-zero, and instant restore." +--- + +# Lakebase Autoscaling + +Patterns and best practices for using Lakebase Autoscaling, the next-generation managed PostgreSQL on Databricks with autoscaling compute, branching, scale-to-zero, and instant restore. + +## When to Use + +Use this skill when: +- Building applications that need a PostgreSQL database with autoscaling compute +- Working with database branching for dev/test/staging workflows +- Adding persistent state to applications with scale-to-zero cost savings +- Implementing reverse ETL from Delta Lake to an operational database via synced tables +- Managing Lakebase Autoscaling projects, branches, computes, or credentials + +## Overview + +Lakebase Autoscaling is Databricks' next-generation managed PostgreSQL service for OLTP workloads. It provides autoscaling compute, Git-like branching, scale-to-zero, and instant point-in-time restore. + +| Feature | Description | +|---------|-------------| +| **Autoscaling Compute** | 0.5-112 CU with 2 GB RAM per CU; scales dynamically based on load | +| **Scale-to-Zero** | Compute suspends after configurable inactivity timeout | +| **Branching** | Create isolated database environments (like Git branches) for dev/test | +| **Instant Restore** | Point-in-time restore from any moment within the configured window (up to 35 days) | +| **OAuth Authentication** | Token-based auth via Databricks SDK (1-hour expiry) | +| **Reverse ETL** | Sync data from Delta tables to PostgreSQL via synced tables | + +**Available Regions (AWS):** us-east-1, us-east-2, eu-central-1, eu-west-1, eu-west-2, ap-south-1, ap-southeast-1, ap-southeast-2 + +**Available Regions (Azure Beta):** eastus2, westeurope, westus + +## Project Hierarchy + +Understanding the hierarchy is essential for working with Lakebase Autoscaling: + +``` +Project (top-level container) + └── Branch(es) (isolated database environments) + ├── Compute (primary R/W endpoint) + ├── Read Replica(s) (optional, read-only) + ├── Role(s) (Postgres roles) + └── Database(s) (Postgres databases) + └── Schema(s) +``` + +| Object | Description | +|--------|-------------| +| **Project** | Top-level container. Created via `w.postgres.create_project()`. | +| **Branch** | Isolated database environment with copy-on-write storage. Default branch is `production`. | +| **Compute** | Postgres server powering a branch. Configurable CU sizing and autoscaling. | +| **Database** | Standard Postgres database within a branch. Default is `databricks_postgres`. | + +## Quick Start + +Create a project and connect: + +```python +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.postgres import Project, ProjectSpec + +w = WorkspaceClient() + +# Create a project (long-running operation) +operation = w.postgres.create_project( + project=Project( + spec=ProjectSpec( + display_name="My Application", + pg_version="17" + ) + ), + project_id="my-app" +) +result = operation.wait() +print(f"Created project: {result.name}") +``` + +## Common Patterns + +### Generate OAuth Token + +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +# Generate database credential for connecting (optionally scoped to an endpoint) +cred = w.postgres.generate_database_credential( + endpoint="projects/my-app/branches/production/endpoints/ep-primary" +) +token = cred.token # Use as password in connection string +# Token expires after 1 hour +``` + +### Connect from Notebook + +```python +import psycopg +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +# Get endpoint details +endpoint = w.postgres.get_endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary" +) +host = endpoint.status.hosts.host + +# Generate token (scoped to endpoint) +cred = w.postgres.generate_database_credential( + endpoint="projects/my-app/branches/production/endpoints/ep-primary" +) + +# Connect using psycopg3 +conn_string = ( + f"host={host} " + f"dbname=databricks_postgres " + f"user={w.current_user.me().user_name} " + f"password={cred.token} " + f"sslmode=require" +) +with psycopg.connect(conn_string) as conn: + with conn.cursor() as cur: + cur.execute("SELECT version()") + print(cur.fetchone()) +``` + +### Create a Branch for Development + +```python +from databricks.sdk.service.postgres import Branch, BranchSpec, Duration + +# Create a dev branch with 7-day expiration +branch = w.postgres.create_branch( + parent="projects/my-app", + branch=Branch( + spec=BranchSpec( + source_branch="projects/my-app/branches/production", + ttl=Duration(seconds=604800) # 7 days + ) + ), + branch_id="development" +).wait() +print(f"Branch created: {branch.name}") +``` + +### Resize Compute (Autoscaling) + +```python +from databricks.sdk.service.postgres import Endpoint, EndpointSpec, FieldMask + +# Update compute to autoscale between 2-8 CU +w.postgres.update_endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary", + endpoint=Endpoint( + name="projects/my-app/branches/production/endpoints/ep-primary", + spec=EndpointSpec( + autoscaling_limit_min_cu=2.0, + autoscaling_limit_max_cu=8.0 + ) + ), + update_mask=FieldMask(field_mask=[ + "spec.autoscaling_limit_min_cu", + "spec.autoscaling_limit_max_cu" + ]) +).wait() +``` + +## MCP Tools + +The following MCP tools are available for managing Lakebase infrastructure. Use `type="autoscale"` for Lakebase Autoscaling. + +### Database (Project) Management + +| Tool | Description | +|------|-------------| +| `create_or_update_lakebase_database` | Create or update a database. Finds by name, creates if new, updates if existing. Use `type="autoscale"`, `display_name`, `pg_version` params. A new project auto-creates a production branch, default compute, and databricks_postgres database. | +| `get_lakebase_database` | Get database details (including branches and endpoints) or list all. Pass `name` to get one, omit to list all. Use `type="autoscale"` to filter. | +| `delete_lakebase_database` | Delete a project and all its branches, computes, and data. Use `type="autoscale"`. | + +### Branch Management + +| Tool | Description | +|------|-------------| +| `create_or_update_lakebase_branch` | Create or update a branch with its compute endpoint. Params: `project_name`, `branch_id`, `source_branch`, `ttl_seconds`, `is_protected`, plus compute params (`autoscaling_limit_min_cu`, `autoscaling_limit_max_cu`, `scale_to_zero_seconds`). | +| `delete_lakebase_branch` | Delete a branch and its compute endpoints. | + +### Credentials + +| Tool | Description | +|------|-------------| +| `generate_lakebase_credential` | Generate OAuth token for PostgreSQL connections (1-hour expiry). Pass `endpoint` resource name for autoscale. | + +## Reference Files + +- [projects.md](projects.md) - Project management patterns and settings +- [branches.md](branches.md) - Branching workflows, protection, and expiration +- [computes.md](computes.md) - Compute sizing, autoscaling, and scale-to-zero +- [connection-patterns.md](connection-patterns.md) - Connection patterns for different use cases +- [reverse-etl.md](reverse-etl.md) - Synced tables from Delta Lake to Lakebase + +## CLI Quick Reference + +```bash +# Create a project +databricks postgres create-project \ + --project-id my-app \ + --json '{"spec": {"display_name": "My App", "pg_version": "17"}}' + +# List projects +databricks postgres list-projects + +# Get project details +databricks postgres get-project projects/my-app + +# Create a branch +databricks postgres create-branch projects/my-app development \ + --json '{"spec": {"source_branch": "projects/my-app/branches/production", "no_expiry": true}}' + +# List branches +databricks postgres list-branches projects/my-app + +# Get endpoint details +databricks postgres get-endpoint projects/my-app/branches/production/endpoints/ep-primary + +# Delete a project +databricks postgres delete-project projects/my-app +``` + +## Key Differences from Lakebase Provisioned + +| Aspect | Provisioned | Autoscaling | +|--------|-------------|-------------| +| SDK module | `w.database` | `w.postgres` | +| Top-level resource | Instance | Project | +| Capacity | CU_1, CU_2, CU_4, CU_8 (16 GB/CU) | 0.5-112 CU (2 GB/CU) | +| Branching | Not supported | Full branching support | +| Scale-to-zero | Not supported | Configurable timeout | +| Operations | Synchronous | Long-running operations (LRO) | +| Read replicas | Readable secondaries | Dedicated read-only endpoints | + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Token expired during long query** | Implement token refresh loop; tokens expire after 1 hour | +| **Connection refused after scale-to-zero** | Compute wakes automatically on connection; reactivation takes a few hundred ms; implement retry logic | +| **DNS resolution fails on macOS** | Use `dig` command to resolve hostname, pass `hostaddr` to psycopg | +| **Branch deletion blocked** | Delete child branches first; cannot delete branches with children | +| **Autoscaling range too wide** | Max - min cannot exceed 8 CU (e.g., 8-16 CU is valid, 0.5-32 CU is not) | +| **SSL required error** | Always use `sslmode=require` in connection string | +| **Update mask required** | All update operations require an `update_mask` specifying fields to modify | +| **Connection closed after 24h idle** | All connections have a 24-hour idle timeout and 3-day max lifetime; implement retry logic | + +## Current Limitations + +These features are NOT yet supported in Lakebase Autoscaling: +- High availability with readable secondaries (use read replicas instead) +- Databricks Apps UI integration (Apps can connect manually via credentials) +- Feature Store integration +- Stateful AI agents (LangChain memory) +- Postgres-to-Delta sync (only Delta-to-Postgres reverse ETL) +- Custom billing tags and serverless budget policies +- Direct migration from Lakebase Provisioned (use pg_dump/pg_restore or reverse ETL) + +## SDK Version Requirements + +- **Databricks SDK for Python**: >= 0.81.0 (for `w.postgres` module) +- **psycopg**: 3.x (supports `hostaddr` parameter for DNS workaround) +- **SQLAlchemy**: 2.x with `postgresql+psycopg` driver + +```python +%pip install -U "databricks-sdk>=0.81.0" "psycopg[binary]>=3.0" sqlalchemy +``` + +## Notes + +- **Compute Units** in Autoscaling provide ~2 GB RAM each (vs 16 GB in Provisioned). +- **Resource naming** follows hierarchical paths: `projects/{id}/branches/{id}/endpoints/{id}`. +- All create/update/delete operations are **long-running** -- use `.wait()` in the SDK. +- Tokens are short-lived (1 hour) -- production apps MUST implement token refresh. +- **Postgres versions** 16 and 17 are supported. + +## Related Skills + +- **[databricks-lakebase-provisioned](../databricks-lakebase-provisioned/SKILL.md)** - fixed-capacity managed PostgreSQL (predecessor) +- **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - full-stack apps that can use Lakebase for persistence +- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python apps with Lakebase backend +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - SDK used for project management and token generation +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying apps with Lakebase resources +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - scheduling reverse ETL sync jobs diff --git a/.claude/skills/databricks-lakebase-autoscale/branches.md b/.claude/skills/databricks-lakebase-autoscale/branches.md new file mode 100644 index 00000000..f44f7234 --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/branches.md @@ -0,0 +1,212 @@ +# Lakebase Autoscaling Branches + +## Overview + +Branches in Lakebase Autoscaling are isolated database environments that share storage with their parent through copy-on-write. They enable Git-like workflows for databases: create isolated dev/test environments, test schema changes safely, and recover from mistakes. + +## Branch Types + +| Option | Description | Use Case | +|--------|-------------|----------| +| **Current data** | Branch from latest state of parent | Development, testing with current data | +| **Past data** | Branch from a specific point in time | Point-in-time recovery, historical analysis | + +## Creating a Branch + +### With Expiration (TTL) + +```python +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.postgres import Branch, BranchSpec, Duration + +w = WorkspaceClient() + +# Create branch with 7-day expiration +result = w.postgres.create_branch( + parent="projects/my-app", + branch=Branch( + spec=BranchSpec( + source_branch="projects/my-app/branches/production", + ttl=Duration(seconds=604800) # 7 days + ) + ), + branch_id="development" +).wait() + +print(f"Branch created: {result.name}") +print(f"Expires: {result.status.expire_time}") +``` + +### Permanent Branch (No Expiration) + +```python +result = w.postgres.create_branch( + parent="projects/my-app", + branch=Branch( + spec=BranchSpec( + source_branch="projects/my-app/branches/production", + no_expiry=True + ) + ), + branch_id="staging" +).wait() +``` + +### CLI + +```bash +# With TTL +databricks postgres create-branch projects/my-app development \ + --json '{ + "spec": { + "source_branch": "projects/my-app/branches/production", + "ttl": "604800s" + } + }' + +# Permanent +databricks postgres create-branch projects/my-app staging \ + --json '{ + "spec": { + "source_branch": "projects/my-app/branches/production", + "no_expiry": true + } + }' +``` + +## Getting Branch Details + +```python +branch = w.postgres.get_branch( + name="projects/my-app/branches/development" +) + +print(f"Branch: {branch.name}") +print(f"Protected: {branch.status.is_protected}") +print(f"Default: {branch.status.default}") +print(f"State: {branch.status.current_state}") +print(f"Size: {branch.status.logical_size_bytes} bytes") +``` + +## Listing Branches + +```python +branches = list(w.postgres.list_branches( + parent="projects/my-app" +)) + +for branch in branches: + print(f"Branch: {branch.name}") + print(f" Default: {branch.status.default}") + print(f" Protected: {branch.status.is_protected}") +``` + +## Protecting a Branch + +Protected branches cannot be deleted, reset, or archived. + +```python +from databricks.sdk.service.postgres import Branch, BranchSpec, FieldMask + +w.postgres.update_branch( + name="projects/my-app/branches/production", + branch=Branch( + name="projects/my-app/branches/production", + spec=BranchSpec(is_protected=True) + ), + update_mask=FieldMask(field_mask=["spec.is_protected"]) +).wait() +``` + +To remove protection: + +```python +w.postgres.update_branch( + name="projects/my-app/branches/production", + branch=Branch( + name="projects/my-app/branches/production", + spec=BranchSpec(is_protected=False) + ), + update_mask=FieldMask(field_mask=["spec.is_protected"]) +).wait() +``` + +## Updating Branch Expiration + +```python +# Extend to 14 days +w.postgres.update_branch( + name="projects/my-app/branches/development", + branch=Branch( + name="projects/my-app/branches/development", + spec=BranchSpec( + is_protected=False, + ttl=Duration(seconds=1209600) # 14 days + ) + ), + update_mask=FieldMask(field_mask=["spec.is_protected", "spec.expiration"]) +).wait() + +# Remove expiration +w.postgres.update_branch( + name="projects/my-app/branches/development", + branch=Branch( + name="projects/my-app/branches/development", + spec=BranchSpec(no_expiry=True) + ), + update_mask=FieldMask(field_mask=["spec.expiration"]) +).wait() +``` + +## Resetting a Branch from Parent + +Reset completely replaces a branch's data and schema with the latest from its parent. Local changes are lost. + +```python +w.postgres.reset_branch( + name="projects/my-app/branches/development" +).wait() +``` + +**Constraints:** +- Root branches (like `production`) cannot be reset (no parent) +- Branches with children cannot be reset (delete children first) +- Connections are temporarily interrupted during reset + +## Deleting a Branch + +```python +w.postgres.delete_branch( + name="projects/my-app/branches/development" +).wait() +``` + +**Constraints:** +- Cannot delete branches with child branches (delete children first) +- Cannot delete protected branches (remove protection first) +- Cannot delete the default branch + +## Branch Expiration + +Branch expiration sets an automatic deletion timestamp. Useful for: +- **CI/CD environments**: 2-4 hours +- **Demos**: 24-48 hours +- **Feature development**: 1-7 days +- **Long-term testing**: up to 30 days + +**Maximum expiration period:** 30 days from current time. + +### Expiration Restrictions + +- Cannot expire protected branches +- Cannot expire default branches +- Cannot expire branches that have children +- When a branch expires, all compute resources are also deleted + +## Best Practices + +1. **Use TTL for ephemeral branches**: Set expiration for dev/test branches to avoid accumulation +2. **Protect production branches**: Prevent accidental deletion or reset +3. **Reset instead of recreate**: Use reset from parent when you need fresh data without new branch overhead +4. **Schema diff before merge**: Compare schemas between branches before applying changes to production +5. **Monitor unarchived limit**: Only 10 unarchived branches are allowed per project diff --git a/.claude/skills/databricks-lakebase-autoscale/computes.md b/.claude/skills/databricks-lakebase-autoscale/computes.md new file mode 100644 index 00000000..0f53d50c --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/computes.md @@ -0,0 +1,208 @@ +# Lakebase Autoscaling Computes + +## Overview + +A compute is a virtualized service that runs Postgres for a branch. Each branch has one primary read-write compute and can have optional read replicas. Computes support autoscaling, scale-to-zero, and granular sizing from 0.5 to 112 CU. + +## Compute Sizing + +Each Compute Unit (CU) allocates approximately 2 GB of RAM. + +### Available Sizes + +| Category | Range | Notes | +|----------|-------|-------| +| **Autoscale computes** | 0.5-32 CU | Dynamic scaling within range (max-min <= 8 CU) | +| **Large fixed-size** | 36-112 CU | Fixed size, no autoscaling | + +### Representative Sizes + +| Compute Units | RAM | Max Connections | +|--------------|-----|-----------------| +| 0.5 CU | ~1 GB | 104 | +| 1 CU | ~2 GB | 209 | +| 4 CU | ~8 GB | 839 | +| 8 CU | ~16 GB | 1,678 | +| 16 CU | ~32 GB | 3,357 | +| 32 CU | ~64 GB | 4,000 | +| 64 CU | ~128 GB | 4,000 | +| 112 CU | ~224 GB | 4,000 | + +**Note:** Lakebase Provisioned used ~16 GB per CU. Autoscaling uses ~2 GB per CU for more granular scaling. + +## Creating a Compute + +```python +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.postgres import Endpoint, EndpointSpec, EndpointType + +w = WorkspaceClient() + +# Create a read-write compute endpoint +result = w.postgres.create_endpoint( + parent="projects/my-app/branches/production", + endpoint=Endpoint( + spec=EndpointSpec( + endpoint_type=EndpointType.ENDPOINT_TYPE_READ_WRITE, + autoscaling_limit_min_cu=0.5, + autoscaling_limit_max_cu=4.0 + ) + ), + endpoint_id="my-compute" +).wait() + +print(f"Endpoint created: {result.name}") +print(f"Host: {result.status.hosts.host}") +``` + +### CLI + +```bash +databricks postgres create-endpoint \ + projects/my-app/branches/production my-compute \ + --json '{ + "spec": { + "endpoint_type": "ENDPOINT_TYPE_READ_WRITE", + "autoscaling_limit_min_cu": 0.5, + "autoscaling_limit_max_cu": 4.0 + } + }' +``` + +**Important:** Each branch can have only one read-write compute. + +## Getting Compute Details + +```python +endpoint = w.postgres.get_endpoint( + name="projects/my-app/branches/production/endpoints/my-compute" +) + +print(f"Endpoint: {endpoint.name}") +print(f"Type: {endpoint.status.endpoint_type}") +print(f"State: {endpoint.status.current_state}") +print(f"Host: {endpoint.status.hosts.host}") +print(f"Min CU: {endpoint.status.autoscaling_limit_min_cu}") +print(f"Max CU: {endpoint.status.autoscaling_limit_max_cu}") +``` + +## Listing Computes + +```python +endpoints = list(w.postgres.list_endpoints( + parent="projects/my-app/branches/production" +)) + +for ep in endpoints: + print(f"Endpoint: {ep.name}") + print(f" Type: {ep.status.endpoint_type}") + print(f" CU Range: {ep.status.autoscaling_limit_min_cu}-{ep.status.autoscaling_limit_max_cu}") +``` + +## Resizing a Compute + +Use `update_mask` to specify which fields to update: + +```python +from databricks.sdk.service.postgres import Endpoint, EndpointSpec, FieldMask + +# Update min and max CU +w.postgres.update_endpoint( + name="projects/my-app/branches/production/endpoints/my-compute", + endpoint=Endpoint( + name="projects/my-app/branches/production/endpoints/my-compute", + spec=EndpointSpec( + autoscaling_limit_min_cu=2.0, + autoscaling_limit_max_cu=8.0 + ) + ), + update_mask=FieldMask(field_mask=[ + "spec.autoscaling_limit_min_cu", + "spec.autoscaling_limit_max_cu" + ]) +).wait() +``` + +### CLI + +```bash +# Update single field +databricks postgres update-endpoint \ + projects/my-app/branches/production/endpoints/my-compute \ + spec.autoscaling_limit_max_cu \ + --json '{"spec": {"autoscaling_limit_max_cu": 8.0}}' + +# Update multiple fields +databricks postgres update-endpoint \ + projects/my-app/branches/production/endpoints/my-compute \ + "spec.autoscaling_limit_min_cu,spec.autoscaling_limit_max_cu" \ + --json '{"spec": {"autoscaling_limit_min_cu": 2.0, "autoscaling_limit_max_cu": 8.0}}' +``` + +## Deleting a Compute + +```python +w.postgres.delete_endpoint( + name="projects/my-app/branches/production/endpoints/my-compute" +).wait() +``` + +## Autoscaling + +Autoscaling dynamically adjusts compute resources based on workload demand. + +### Configuration + +- **Range:** 0.5-32 CU +- **Constraint:** Max - Min cannot exceed 8 CU +- **Valid examples:** 4-8 CU, 8-16 CU, 16-24 CU +- **Invalid example:** 0.5-32 CU (range of 31.5 CU) + +### Best Practices + +- Set minimum CU large enough to cache your working set in memory +- Performance may be degraded until compute scales up and caches data +- Connection limits are based on the maximum CU in the range + +## Scale-to-Zero + +Automatically suspends compute after a period of inactivity. + +| Setting | Description | +|---------|-------------| +| **Enabled** | Compute suspends after inactivity timeout (saves cost) | +| **Disabled** | Always-active compute (eliminates wake-up latency) | + +**Default behavior:** +- `production` branch: Scale-to-zero **disabled** (always active) +- Other branches: Scale-to-zero can be configured + +**Default inactivity timeout:** 5 minutes +**Minimum inactivity timeout:** 60 seconds + +### Wake-up Behavior + +When a connection arrives on a suspended compute: +1. Compute starts automatically (reactivation takes a few hundred milliseconds) +2. The connection request is handled transparently once active +3. Compute restarts at minimum autoscaling size (if autoscaling enabled) +4. Applications should implement connection retry logic for the brief reactivation period + +### Session Context After Reactivation + +When a compute suspends and reactivates, session context is **reset**: +- In-memory statistics and cache contents are cleared +- Temporary tables and prepared statements are lost +- Session-specific configuration settings reset +- Connection pools and active transactions are terminated + +If your application requires persistent session data, consider disabling scale-to-zero. + +## Sizing Guidance + +| Factor | Recommendation | +|--------|---------------| +| Query complexity | Complex analytical queries benefit from larger computes | +| Concurrent connections | More connections need more CPU and memory | +| Data volume | Larger datasets may need more memory for performance | +| Response time | Critical apps may require larger computes | diff --git a/.claude/skills/databricks-lakebase-autoscale/connection-patterns.md b/.claude/skills/databricks-lakebase-autoscale/connection-patterns.md new file mode 100644 index 00000000..398862b3 --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/connection-patterns.md @@ -0,0 +1,304 @@ +# Lakebase Autoscaling Connection Patterns + +## Overview + +This document covers different connection patterns for Lakebase Autoscaling, from simple scripts to production applications with token refresh. + +## Authentication Methods + +Lakebase Autoscaling supports two authentication methods: + +| Method | Token Lifetime | Best For | +|--------|---------------|----------| +| **OAuth tokens** | 1 hour (must refresh) | Interactive sessions, workspace-integrated apps | +| **Native Postgres passwords** | No expiry | Long-running processes, tools without token rotation | + +**Connection timeouts (both methods):** +- **24-hour idle timeout**: Connections with no activity for 24 hours are automatically closed +- **3-day maximum connection life**: Connections alive for more than 3 days may be closed + +Design your applications to handle connection timeouts with retry logic. + +## Connection Methods + +### 1. Direct psycopg Connection (Simple Scripts) + +For one-off scripts or notebooks: + +```python +import psycopg +from databricks.sdk import WorkspaceClient + +def get_connection(project_id: str, branch_id: str = "production", + endpoint_id: str = None, database_name: str = "databricks_postgres"): + """Get a database connection with fresh OAuth token.""" + w = WorkspaceClient() + + # Get endpoint details to find the host + if endpoint_id: + ep_name = f"projects/{project_id}/branches/{branch_id}/endpoints/{endpoint_id}" + else: + # List endpoints and pick the primary R/W one + endpoints = list(w.postgres.list_endpoints( + parent=f"projects/{project_id}/branches/{branch_id}" + )) + ep_name = endpoints[0].name + + endpoint = w.postgres.get_endpoint(name=ep_name) + host = endpoint.status.hosts.host + + # Generate OAuth token (valid for 1 hour) + cred = w.postgres.generate_database_credential(endpoint=ep_name) + + # Build connection string + conn_string = ( + f"host={host} " + f"dbname={database_name} " + f"user={w.current_user.me().user_name} " + f"password={cred.token} " + f"sslmode=require" + ) + + return psycopg.connect(conn_string) + +# Usage +with get_connection("my-app") as conn: + with conn.cursor() as cur: + cur.execute("SELECT NOW()") + print(cur.fetchone()) +``` + +### 2. Connection Pool with Token Refresh (Production) + +For long-running applications that need connection pooling: + +```python +import asyncio +import uuid +from contextlib import asynccontextmanager +from typing import AsyncGenerator, Optional + +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker +from databricks.sdk import WorkspaceClient + + +class LakebaseAutoscaleConnectionManager: + """Manages Lakebase Autoscaling connections with automatic token refresh.""" + + def __init__( + self, + project_id: str, + branch_id: str = "production", + database_name: str = "databricks_postgres", + pool_size: int = 5, + max_overflow: int = 10, + token_refresh_seconds: int = 3000 # 50 minutes + ): + self.project_id = project_id + self.branch_id = branch_id + self.database_name = database_name + self.pool_size = pool_size + self.max_overflow = max_overflow + self.token_refresh_seconds = token_refresh_seconds + + self._current_token: Optional[str] = None + self._refresh_task: Optional[asyncio.Task] = None + self._engine = None + self._session_maker = None + + def _generate_token(self) -> str: + """Generate fresh OAuth token.""" + w = WorkspaceClient() + # Get primary endpoint name for token scoping + endpoints = list(w.postgres.list_endpoints( + parent=f"projects/{self.project_id}/branches/{self.branch_id}" + )) + endpoint_name = endpoints[0].name if endpoints else None + cred = w.postgres.generate_database_credential(endpoint=endpoint_name) + return cred.token + + def _get_host(self) -> str: + """Get the connection host from the primary endpoint.""" + w = WorkspaceClient() + endpoints = list(w.postgres.list_endpoints( + parent=f"projects/{self.project_id}/branches/{self.branch_id}" + )) + if not endpoints: + raise RuntimeError( + f"No endpoints found for projects/{self.project_id}/branches/{self.branch_id}" + ) + endpoint = w.postgres.get_endpoint(name=endpoints[0].name) + return endpoint.status.hosts.host + + async def _refresh_loop(self): + """Background task to refresh token periodically.""" + while True: + await asyncio.sleep(self.token_refresh_seconds) + try: + self._current_token = await asyncio.to_thread(self._generate_token) + except Exception as e: + print(f"Token refresh failed: {e}") + + def initialize(self): + """Initialize database engine and start token refresh.""" + w = WorkspaceClient() + + # Get host info + host = self._get_host() + username = w.current_user.me().user_name + + # Generate initial token + self._current_token = self._generate_token() + + # Create engine (password injected via event) + url = ( + f"postgresql+psycopg://{username}@" + f"{host}:5432/{self.database_name}" + ) + + self._engine = create_async_engine( + url, + pool_size=self.pool_size, + max_overflow=self.max_overflow, + pool_recycle=3600, + connect_args={"sslmode": "require"} + ) + + # Inject token on connect + @event.listens_for(self._engine.sync_engine, "do_connect") + def inject_token(dialect, conn_rec, cargs, cparams): + cparams["password"] = self._current_token + + self._session_maker = async_sessionmaker( + self._engine, + class_=AsyncSession, + expire_on_commit=False + ) + + def start_refresh(self): + """Start background token refresh task.""" + if not self._refresh_task: + self._refresh_task = asyncio.create_task(self._refresh_loop()) + + async def stop_refresh(self): + """Stop token refresh task.""" + if self._refresh_task: + self._refresh_task.cancel() + try: + await self._refresh_task + except asyncio.CancelledError: + pass + self._refresh_task = None + + @asynccontextmanager + async def session(self) -> AsyncGenerator[AsyncSession, None]: + """Get a database session.""" + async with self._session_maker() as session: + yield session + + async def close(self): + """Close all connections.""" + await self.stop_refresh() + if self._engine: + await self._engine.dispose() + + +# Usage in FastAPI +from fastapi import FastAPI + +app = FastAPI() +db_manager = LakebaseAutoscaleConnectionManager("my-app", "production", "my_database") + +@app.on_event("startup") +async def startup(): + db_manager.initialize() + db_manager.start_refresh() + +@app.on_event("shutdown") +async def shutdown(): + await db_manager.close() + +@app.get("/data") +async def get_data(): + async with db_manager.session() as session: + result = await session.execute("SELECT * FROM my_table") + return result.fetchall() +``` + +### 3. Static URL Mode (Local Development) + +For local development, use a static connection URL: + +```python +import os +from sqlalchemy.ext.asyncio import create_async_engine + +# Set environment variable with full connection URL +# LAKEBASE_PG_URL=postgresql://user:password@host:5432/database + +def get_database_url() -> str: + """Get database URL from environment.""" + url = os.environ.get("LAKEBASE_PG_URL") + if url and url.startswith("postgresql://"): + # Convert to psycopg3 async driver + url = url.replace("postgresql://", "postgresql+psycopg://", 1) + return url + +engine = create_async_engine( + get_database_url(), + pool_size=5, + connect_args={"sslmode": "require"} +) +``` + +### 4. DNS Resolution Workaround (macOS) + +Python's `socket.getaddrinfo()` fails with long hostnames on macOS. Use `dig` as fallback: + +```python +import subprocess +import socket + +def resolve_hostname(hostname: str) -> str: + """Resolve hostname using dig command (macOS workaround).""" + try: + return socket.gethostbyname(hostname) + except socket.gaierror: + pass + + try: + result = subprocess.run( + ["dig", "+short", hostname], + capture_output=True, text=True, timeout=5 + ) + ips = result.stdout.strip().split('\n') + for ip in ips: + if ip and not ip.startswith(';'): + return ip + except Exception: + pass + + raise RuntimeError(f"Could not resolve hostname: {hostname}") + +# Use with psycopg +conn_params = { + "host": hostname, # For TLS SNI + "hostaddr": resolve_hostname(hostname), # Actual IP + "dbname": database_name, + "user": username, + "password": token, + "sslmode": "require" +} +conn = psycopg.connect(**conn_params) +``` + +## Best Practices + +1. **Always use SSL**: Set `sslmode=require` in all connections +2. **Implement token refresh**: Tokens expire after 1 hour; refresh at 50 minutes +3. **Use connection pooling**: Avoid creating new connections per request +4. **Handle DNS issues on macOS**: Use the `hostaddr` workaround if needed +5. **Close connections properly**: Use context managers or explicit cleanup +6. **Handle scale-to-zero wake-up**: First connection after idle may take 2-5 seconds +7. **Log token refresh events**: Helps debug authentication issues diff --git a/.claude/skills/databricks-lakebase-autoscale/projects.md b/.claude/skills/databricks-lakebase-autoscale/projects.md new file mode 100644 index 00000000..659207a4 --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/projects.md @@ -0,0 +1,204 @@ +# Lakebase Autoscaling Projects + +## Overview + +A project is the top-level container for Lakebase Autoscaling resources, including branches, computes, databases, and roles. Each project is isolated and contains its own Postgres version, compute defaults, and restore window settings. + +## Project Structure + +``` +Project + └── Branches (production, development, staging, etc.) + ├── Computes (R/W compute, read replicas) + ├── Roles (Postgres roles) + └── Databases (Postgres databases) +``` + +When a project is created, it includes by default: +- A `production` branch (the default branch) +- A primary read-write compute (8-32 CU, autoscaling enabled, scale-to-zero disabled) +- A `databricks_postgres` database +- A Postgres role for the creating user's Databricks identity + +## Resource Naming + +Projects follow a hierarchical naming convention: +``` +projects/{project_id} +``` + +**Resource ID requirements:** +- 1-63 characters long +- Lowercase letters, digits, and hyphens only +- Cannot start or end with a hyphen +- Cannot be changed after creation + +## Creating a Project + +### Python SDK + +```python +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.postgres import Project, ProjectSpec + +w = WorkspaceClient() + +# Create a project (long-running operation) +operation = w.postgres.create_project( + project=Project( + spec=ProjectSpec( + display_name="My Application", + pg_version="17" + ) + ), + project_id="my-app" +) + +# Wait for completion +result = operation.wait() +print(f"Created project: {result.name}") +print(f"Display name: {result.status.display_name}") +print(f"Postgres version: {result.status.pg_version}") +``` + +### CLI + +```bash +databricks postgres create-project \ + --project-id my-app \ + --json '{ + "spec": { + "display_name": "My Application", + "pg_version": "17" + } + }' +``` + +## Getting Project Details + +### Python SDK + +```python +project = w.postgres.get_project(name="projects/my-app") + +print(f"Project: {project.name}") +print(f"Display name: {project.status.display_name}") +print(f"Postgres version: {project.status.pg_version}") +``` + +### CLI + +```bash +databricks postgres get-project projects/my-app +``` + +**Note:** The `spec` field is not populated for GET operations. All properties are returned in the `status` field. + +## Listing Projects + +```python +projects = w.postgres.list_projects() + +for project in projects: + print(f"Project: {project.name}") + print(f" Display name: {project.status.display_name}") + print(f" Postgres version: {project.status.pg_version}") +``` + +## Updating a Project + +Updates require an `update_mask` specifying which fields to modify: + +```python +from databricks.sdk.service.postgres import Project, ProjectSpec, FieldMask + +# Update display name +operation = w.postgres.update_project( + name="projects/my-app", + project=Project( + name="projects/my-app", + spec=ProjectSpec( + display_name="My Updated Application" + ) + ), + update_mask=FieldMask(field_mask=["spec.display_name"]) +) +result = operation.wait() +``` + +### CLI + +```bash +databricks postgres update-project projects/my-app spec.display_name \ + --json '{ + "spec": { + "display_name": "My Updated Application" + } + }' +``` + +## Deleting a Project + +**WARNING:** Deleting a project is permanent and also deletes all branches, computes, databases, roles, and data. + +Delete all Unity Catalog catalogs and synced tables before deleting the project. + +```python +operation = w.postgres.delete_project(name="projects/my-app") +# This is a long-running operation +``` + +### CLI + +```bash +databricks postgres delete-project projects/my-app +``` + +## Project Settings + +### Compute Defaults + +Default settings for new primary computes: +- Compute size range (0.5-112 CU) +- Scale-to-zero timeout (default: 5 minutes) + +### Instant Restore + +Configure the restore window length (2-35 days). Longer windows increase storage costs. + +### Postgres Version + +Supports Postgres 16 and Postgres 17. + +## Project Limits + +| Resource | Limit | +|----------|-------| +| Concurrently active computes | 20 | +| Branches per project | 500 | +| Postgres roles per branch | 500 | +| Postgres databases per branch | 500 | +| Logical data size per branch | 8 TB | +| Projects per workspace | 1000 | +| Protected branches | 1 | +| Root branches | 3 | +| Unarchived branches | 10 | +| Snapshots | 10 | +| Maximum history retention | 35 days | +| Minimum scale-to-zero time | 60 seconds | + +## Long-Running Operations + +All create, update, and delete operations return a long-running operation (LRO). Use `.wait()` in the SDK to block until completion: + +```python +# Start operation +operation = w.postgres.create_project(...) + +# Wait for completion +result = operation.wait() + +# Or check status manually +op_status = w.postgres.get_operation(name=operation.name) +print(f"Done: {op_status.done}") +``` diff --git a/.claude/skills/databricks-lakebase-autoscale/reverse-etl.md b/.claude/skills/databricks-lakebase-autoscale/reverse-etl.md new file mode 100644 index 00000000..f983eebb --- /dev/null +++ b/.claude/skills/databricks-lakebase-autoscale/reverse-etl.md @@ -0,0 +1,177 @@ +# Reverse ETL with Lakebase Autoscaling + +## Overview + +Reverse ETL allows you to sync data from Unity Catalog Delta tables into Lakebase Autoscaling as PostgreSQL tables. This enables OLTP access patterns on data processed in the Lakehouse. + +## How It Works + +Synced tables create a managed copy of Unity Catalog data in Lakebase: + +1. A new Unity Catalog table (read-only, managed by the sync pipeline) +2. A Postgres table in Lakebase (queryable by applications) + +The sync pipeline uses managed Lakeflow Spark Declarative Pipelines to continuously update both tables. + +### Performance + +- **Continuous writes:** ~1,200 rows/sec per CU +- **Bulk writes:** ~15,000 rows/sec per CU +- **Connections used:** Up to 16 per synced table + +## Sync Modes + +| Mode | Description | Best For | Notes | +|------|-------------|----------|-------| +| **Snapshot** | One-time full copy | Initial setup, historical analysis | 10x more efficient if modifying >10% of data | +| **Triggered** | Scheduled updates on demand | Dashboards updated hourly/daily | Requires CDF on source table | +| **Continuous** | Real-time streaming (seconds of latency) | Live applications | Highest cost, minimum 15s intervals, requires CDF | + +**Note:** Triggered and Continuous modes require Change Data Feed (CDF) enabled on the source table: + +```sql +ALTER TABLE your_catalog.your_schema.your_table +SET TBLPROPERTIES (delta.enableChangeDataFeed = true) +``` + +## Creating Synced Tables + +### Using Python SDK + +```python +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.database import ( + SyncedDatabaseTable, + SyncedTableSpec, + NewPipelineSpec, + SyncedTableSchedulingPolicy, +) + +w = WorkspaceClient() + +# Create a synced table +synced_table = w.database.create_synced_database_table( + SyncedDatabaseTable( + name="lakebase_catalog.schema.synced_table", + spec=SyncedTableSpec( + source_table_full_name="analytics.gold.user_profiles", + primary_key_columns=["user_id"], + scheduling_policy=SyncedTableSchedulingPolicy.TRIGGERED, + new_pipeline_spec=NewPipelineSpec( + storage_catalog="lakebase_catalog", + storage_schema="staging" + ) + ), + ) +) +print(f"Created synced table: {synced_table.name}") +``` + +### Using CLI + +```bash +databricks database create-synced-database-table \ + --json '{ + "name": "lakebase_catalog.schema.synced_table", + "spec": { + "source_table_full_name": "analytics.gold.user_profiles", + "primary_key_columns": ["user_id"], + "scheduling_policy": "TRIGGERED", + "new_pipeline_spec": { + "storage_catalog": "lakebase_catalog", + "storage_schema": "staging" + } + } + }' +``` + +## Checking Synced Table Status + +```python +status = w.database.get_synced_database_table(name="lakebase_catalog.schema.synced_table") +print(f"State: {status.data_synchronization_status.detailed_state}") +print(f"Message: {status.data_synchronization_status.message}") +``` + +## Deleting a Synced Table + +Delete from both Unity Catalog and Postgres: + +1. **Unity Catalog:** Delete from Catalog Explorer or SDK +2. **Postgres:** Drop the table to free storage + +```sql +DROP TABLE your_database.your_schema.your_table; +``` + +## Data Type Mapping + +| Unity Catalog Type | Postgres Type | +|-------------------|---------------| +| BIGINT | BIGINT | +| BINARY | BYTEA | +| BOOLEAN | BOOLEAN | +| DATE | DATE | +| DECIMAL(p,s) | NUMERIC | +| DOUBLE | DOUBLE PRECISION | +| FLOAT | REAL | +| INT | INTEGER | +| INTERVAL | INTERVAL | +| SMALLINT | SMALLINT | +| STRING | TEXT | +| TIMESTAMP | TIMESTAMP WITH TIME ZONE | +| TIMESTAMP_NTZ | TIMESTAMP WITHOUT TIME ZONE | +| TINYINT | SMALLINT | +| ARRAY | JSONB | +| MAP | JSONB | +| STRUCT | JSONB | + +**Unsupported types:** GEOGRAPHY, GEOMETRY, VARIANT, OBJECT + +## Capacity Planning + +- **Connection usage:** Each synced table uses up to 16 connections +- **Size limits:** 2 TB total across all synced tables; recommend < 1 TB per table +- **Naming:** Database, schema, and table names only allow `[A-Za-z0-9_]+` +- **Schema evolution:** Only additive changes (e.g., adding columns) for Triggered/Continuous modes + +## Use Cases + +### Product Catalog for Web App + +```python +w.database.create_synced_database_table( + SyncedDatabaseTable( + name="ecommerce_catalog.public.products", + spec=SyncedTableSpec( + source_table_full_name="gold.products.catalog", + primary_key_columns=["product_id"], + scheduling_policy=SyncedTableSchedulingPolicy.TRIGGERED, + ), + ) +) +``` + +### Real-time Feature Serving + +```python +w.database.create_synced_database_table( + SyncedDatabaseTable( + name="ml_catalog.public.user_features", + spec=SyncedTableSpec( + source_table_full_name="ml.features.user_features", + primary_key_columns=["user_id"], + scheduling_policy=SyncedTableSchedulingPolicy.CONTINUOUS, + ), + ) +) +``` + +## Best Practices + +1. **Enable CDF** on source tables before creating Triggered or Continuous synced tables +2. **Choose appropriate sync mode**: Snapshot for small tables, Triggered for hourly/daily, Continuous for real-time +3. **Monitor sync status**: Check for failures and latency via Catalog Explorer +4. **Index target tables**: Create appropriate indexes in Postgres for your query patterns +5. **Handle schema changes**: Only additive changes are supported for streaming modes +6. **Account for connection limits**: Each synced table uses up to 16 connections diff --git a/.claude/skills/databricks-lakebase-provisioned/SKILL.md b/.claude/skills/databricks-lakebase-provisioned/SKILL.md new file mode 100644 index 00000000..b2b404aa --- /dev/null +++ b/.claude/skills/databricks-lakebase-provisioned/SKILL.md @@ -0,0 +1,308 @@ +--- +name: databricks-lakebase-provisioned +description: "Patterns and best practices for using Lakebase Provisioned (Databricks managed PostgreSQL) for OLTP workloads." +--- + +# Lakebase Provisioned + +Patterns and best practices for using Lakebase Provisioned (Databricks managed PostgreSQL) for OLTP workloads. + +## When to Use + +Use this skill when: +- Building applications that need a PostgreSQL database for transactional workloads +- Adding persistent state to Databricks Apps +- Implementing reverse ETL from Delta Lake to an operational database +- Storing chat/agent memory for LangChain applications + +## Overview + +Lakebase Provisioned is Databricks' managed PostgreSQL database service for OLTP (Online Transaction Processing) workloads. It provides a fully managed PostgreSQL-compatible database that integrates with Unity Catalog and supports OAuth token-based authentication. + +| Feature | Description | +|---------|-------------| +| **Managed PostgreSQL** | Fully managed instances with automatic provisioning | +| **OAuth Authentication** | Token-based auth via Databricks SDK (1-hour expiry) | +| **Unity Catalog** | Register databases for governance | +| **Reverse ETL** | Sync data from Delta tables to PostgreSQL | +| **Apps Integration** | First-class support in Databricks Apps | + +**Available Regions (AWS):** us-east-1, us-east-2, us-west-2, eu-central-1, eu-west-1, ap-south-1, ap-southeast-1, ap-southeast-2 + +## Quick Start + +Create and connect to a Lakebase Provisioned instance: + +```python +from databricks.sdk import WorkspaceClient +import uuid + +# Initialize client +w = WorkspaceClient() + +# Create a database instance +instance = w.database.create_database_instance( + name="my-lakebase-instance", + capacity="CU_1", # CU_1, CU_2, CU_4, CU_8 + stopped=False +) +print(f"Instance created: {instance.name}") +print(f"DNS endpoint: {instance.read_write_dns}") +``` + +## Common Patterns + +### Generate OAuth Token + +```python +from databricks.sdk import WorkspaceClient +import uuid + +w = WorkspaceClient() + +# Generate OAuth token for database connection +cred = w.database.generate_database_credential( + request_id=str(uuid.uuid4()), + instance_names=["my-lakebase-instance"] +) +token = cred.token # Use this as password in connection string +``` + +### Connect from Notebook + +```python +import psycopg +from databricks.sdk import WorkspaceClient +import uuid + +# Get instance details +w = WorkspaceClient() +instance = w.database.get_database_instance(name="my-lakebase-instance") + +# Generate token +cred = w.database.generate_database_credential( + request_id=str(uuid.uuid4()), + instance_names=["my-lakebase-instance"] +) + +# Connect using psycopg3 +conn_string = f"host={instance.read_write_dns} dbname=postgres user={w.current_user.me().user_name} password={cred.token} sslmode=require" +with psycopg.connect(conn_string) as conn: + with conn.cursor() as cur: + cur.execute("SELECT version()") + print(cur.fetchone()) +``` + +### SQLAlchemy with Token Refresh (Production) + +For long-running applications, tokens must be refreshed (expire after 1 hour): + +```python +import asyncio +import os +import uuid +from sqlalchemy import event +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import sessionmaker +from databricks.sdk import WorkspaceClient + +# Token refresh state +_current_token = None +_token_refresh_task = None +TOKEN_REFRESH_INTERVAL = 50 * 60 # 50 minutes (before 1-hour expiry) + +def _generate_token(instance_name: str) -> str: + """Generate fresh OAuth token.""" + w = WorkspaceClient() + cred = w.database.generate_database_credential( + request_id=str(uuid.uuid4()), + instance_names=[instance_name] + ) + return cred.token + +async def _token_refresh_loop(instance_name: str): + """Background task to refresh token every 50 minutes.""" + global _current_token + while True: + await asyncio.sleep(TOKEN_REFRESH_INTERVAL) + _current_token = await asyncio.to_thread(_generate_token, instance_name) + +def init_database(instance_name: str, database_name: str, username: str) -> AsyncEngine: + """Initialize database with OAuth token injection.""" + global _current_token + + w = WorkspaceClient() + instance = w.database.get_database_instance(name=instance_name) + + # Generate initial token + _current_token = _generate_token(instance_name) + + # Build URL (password injected via do_connect) + url = f"postgresql+psycopg://{username}@{instance.read_write_dns}:5432/{database_name}" + + engine = create_async_engine( + url, + pool_size=5, + max_overflow=10, + pool_recycle=3600, + connect_args={"sslmode": "require"} + ) + + # Inject token on each connection + @event.listens_for(engine.sync_engine, "do_connect") + def provide_token(dialect, conn_rec, cargs, cparams): + cparams["password"] = _current_token + + return engine +``` + +### Databricks Apps Integration + +For Databricks Apps, use environment variables for configuration: + +```python +# Environment variables set by Databricks Apps: +# - LAKEBASE_INSTANCE_NAME: Instance name +# - LAKEBASE_DATABASE_NAME: Database name +# - LAKEBASE_USERNAME: Username (optional, defaults to service principal) + +import os + +def is_lakebase_configured() -> bool: + """Check if Lakebase is configured for this app.""" + return bool( + os.environ.get("LAKEBASE_PG_URL") or + (os.environ.get("LAKEBASE_INSTANCE_NAME") and + os.environ.get("LAKEBASE_DATABASE_NAME")) + ) +``` + +Add Lakebase as an app resource via CLI: + +```bash +databricks apps add-resource $APP_NAME \ + --resource-type database \ + --resource-name lakebase \ + --database-instance my-lakebase-instance +``` + +### Register with Unity Catalog + +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +# Register database in Unity Catalog +w.database.register_database_instance( + name="my-lakebase-instance", + catalog="my_catalog", + schema="my_schema" +) +``` + +### MLflow Model Resources + +Declare Lakebase as a model resource for automatic credential provisioning: + +```python +from mlflow.models.resources import DatabricksLakebase + +resources = [ + DatabricksLakebase(database_instance_name="my-lakebase-instance"), +] + +# When logging model +mlflow.langchain.log_model( + model, + artifact_path="model", + resources=resources, + pip_requirements=["databricks-langchain[memory]"] +) +``` + +## MCP Tools + +The following MCP tools are available for managing Lakebase infrastructure. Use `type="provisioned"` for Lakebase Provisioned. + +### Database Management + +| Tool | Description | +|------|-------------| +| `create_or_update_lakebase_database` | Create or update a database. Finds by name, creates if new, updates if existing. Use `type="provisioned"`, `capacity` (CU_1-CU_8), `stopped` params. | +| `get_lakebase_database` | Get database details or list all. Pass `name` to get one, omit to list all. Use `type="provisioned"` to filter. | +| `delete_lakebase_database` | Delete a database and its resources. Use `type="provisioned"`, `force=True` to cascade. | +| `generate_lakebase_credential` | Generate OAuth token for PostgreSQL connections (1-hour expiry). Pass `instance_names` for provisioned. | + +### Reverse ETL (Catalog + Synced Tables) + +| Tool | Description | +|------|-------------| +| `create_or_update_lakebase_sync` | Set up reverse ETL: ensures UC catalog registration exists, then creates a synced table from Delta to Lakebase. Params: `instance_name`, `source_table_name`, `target_table_name`, `scheduling_policy` ("TRIGGERED"/"SNAPSHOT"/"CONTINUOUS"). | +| `delete_lakebase_sync` | Remove a synced table and optionally its UC catalog registration. | + +## Reference Files + +- [connection-patterns.md](connection-patterns.md) - Detailed connection patterns for different use cases +- [reverse-etl.md](reverse-etl.md) - Syncing data from Delta Lake to Lakebase + +## CLI Quick Reference + +```bash +# Create instance +databricks database create-database-instance \ + --name my-lakebase-instance \ + --capacity CU_1 + +# Get instance details +databricks database get-database-instance --name my-lakebase-instance + +# Generate credentials +databricks database generate-database-credential \ + --request-id $(uuidgen) \ + --json '{"instance_names": ["my-lakebase-instance"]}' + +# List instances +databricks database list-database-instances + +# Stop instance (saves cost) +databricks database stop-database-instance --name my-lakebase-instance + +# Start instance +databricks database start-database-instance --name my-lakebase-instance +``` + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Token expired during long query** | Implement token refresh loop (see SQLAlchemy with Token Refresh section); tokens expire after 1 hour | +| **DNS resolution fails on macOS** | Use `dig` command to resolve hostname, pass `hostaddr` to psycopg | +| **Connection refused** | Ensure instance is not stopped; check `instance.state` | +| **Permission denied** | User must be granted access to the Lakebase instance | +| **SSL required error** | Always use `sslmode=require` in connection string | + +## SDK Version Requirements + +- **Databricks SDK for Python**: >= 0.61.0 (0.81.0+ recommended for full API support) +- **psycopg**: 3.x (supports `hostaddr` parameter for DNS workaround) +- **SQLAlchemy**: 2.x with `postgresql+psycopg` driver + +```python +%pip install -U "databricks-sdk>=0.81.0" "psycopg[binary]>=3.0" sqlalchemy +``` + +## Notes + +- **Capacity values** use compute unit sizing: `CU_1`, `CU_2`, `CU_4`, `CU_8`. +- **Lakebase Autoscaling** is a newer offering with automatic scaling but limited regional availability. This skill focuses on **Lakebase Provisioned** which is more widely available. +- For memory/state in LangChain agents, use `databricks-langchain[memory]` which includes Lakebase support. +- Tokens are short-lived (1 hour) - production apps MUST implement token refresh. + +## Related Skills + +- **[databricks-app-apx](../databricks-app-apx/SKILL.md)** - full-stack apps that can use Lakebase for persistence +- **[databricks-app-python](../databricks-app-python/SKILL.md)** - Python apps with Lakebase backend +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - SDK used for instance management and token generation +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying apps with Lakebase resources +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - scheduling reverse ETL sync jobs diff --git a/.claude/skills/databricks-lakebase-provisioned/connection-patterns.md b/.claude/skills/databricks-lakebase-provisioned/connection-patterns.md new file mode 100644 index 00000000..e6843548 --- /dev/null +++ b/.claude/skills/databricks-lakebase-provisioned/connection-patterns.md @@ -0,0 +1,279 @@ +# Lakebase Connection Patterns + +## Overview + +This document covers different connection patterns for Lakebase Provisioned, from simple scripts to production applications with token refresh. + +## Connection Methods + +### 1. Direct psycopg Connection (Simple Scripts) + +For one-off scripts or notebooks: + +```python +import psycopg +from databricks.sdk import WorkspaceClient +import uuid + +def get_connection(instance_name: str, database_name: str = "postgres"): + """Get a database connection with fresh OAuth token.""" + w = WorkspaceClient() + + # Get instance details + instance = w.database.get_database_instance(name=instance_name) + + # Generate OAuth token (valid for 1 hour) + cred = w.database.generate_database_credential( + request_id=str(uuid.uuid4()), + instance_names=[instance_name] + ) + + # Build connection string + conn_string = ( + f"host={instance.read_write_dns} " + f"dbname={database_name} " + f"user={w.current_user.me().user_name} " + f"password={cred.token} " + f"sslmode=require" + ) + + return psycopg.connect(conn_string) + +# Usage +with get_connection("my-instance") as conn: + with conn.cursor() as cur: + cur.execute("SELECT NOW()") + print(cur.fetchone()) +``` + +### 2. Connection Pool with Token Refresh (Production) + +For long-running applications that need connection pooling: + +```python +import asyncio +import uuid +from contextlib import asynccontextmanager +from typing import AsyncGenerator, Optional + +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker +from databricks.sdk import WorkspaceClient + +class LakebaseConnectionManager: + """Manages Lakebase connections with automatic token refresh.""" + + def __init__( + self, + instance_name: str, + database_name: str, + pool_size: int = 5, + max_overflow: int = 10, + token_refresh_seconds: int = 3000 # 50 minutes + ): + self.instance_name = instance_name + self.database_name = database_name + self.pool_size = pool_size + self.max_overflow = max_overflow + self.token_refresh_seconds = token_refresh_seconds + + self._current_token: Optional[str] = None + self._refresh_task: Optional[asyncio.Task] = None + self._engine = None + self._session_maker = None + + def _generate_token(self) -> str: + """Generate fresh OAuth token.""" + w = WorkspaceClient() + cred = w.database.generate_database_credential( + request_id=str(uuid.uuid4()), + instance_names=[self.instance_name] + ) + return cred.token + + async def _refresh_loop(self): + """Background task to refresh token periodically.""" + while True: + await asyncio.sleep(self.token_refresh_seconds) + try: + self._current_token = await asyncio.to_thread(self._generate_token) + except Exception as e: + print(f"Token refresh failed: {e}") + + def initialize(self): + """Initialize database engine and start token refresh.""" + w = WorkspaceClient() + + # Get instance info + instance = w.database.get_database_instance(name=self.instance_name) + username = w.current_user.me().user_name + + # Generate initial token + self._current_token = self._generate_token() + + # Create engine (password injected via event) + url = ( + f"postgresql+psycopg://{username}@" + f"{instance.read_write_dns}:5432/{self.database_name}" + ) + + self._engine = create_async_engine( + url, + pool_size=self.pool_size, + max_overflow=self.max_overflow, + pool_recycle=3600, + connect_args={"sslmode": "require"} + ) + + # Inject token on connect + @event.listens_for(self._engine.sync_engine, "do_connect") + def inject_token(dialect, conn_rec, cargs, cparams): + cparams["password"] = self._current_token + + self._session_maker = async_sessionmaker( + self._engine, + class_=AsyncSession, + expire_on_commit=False + ) + + def start_refresh(self): + """Start background token refresh task.""" + if not self._refresh_task: + self._refresh_task = asyncio.create_task(self._refresh_loop()) + + async def stop_refresh(self): + """Stop token refresh task.""" + if self._refresh_task: + self._refresh_task.cancel() + try: + await self._refresh_task + except asyncio.CancelledError: + pass + self._refresh_task = None + + @asynccontextmanager + async def session(self) -> AsyncGenerator[AsyncSession, None]: + """Get a database session.""" + async with self._session_maker() as session: + yield session + + async def close(self): + """Close all connections.""" + await self.stop_refresh() + if self._engine: + await self._engine.dispose() + +# Usage in FastAPI +from fastapi import FastAPI + +app = FastAPI() +db_manager = LakebaseConnectionManager("my-instance", "my_database") + +@app.on_event("startup") +async def startup(): + db_manager.initialize() + db_manager.start_refresh() + +@app.on_event("shutdown") +async def shutdown(): + await db_manager.close() + +@app.get("/data") +async def get_data(): + async with db_manager.session() as session: + result = await session.execute("SELECT * FROM my_table") + return result.fetchall() +``` + +### 3. Static URL Mode (Local Development) + +For local development, use a static connection URL: + +```python +import os +from sqlalchemy.ext.asyncio import create_async_engine + +# Set environment variable with full connection URL +# LAKEBASE_PG_URL=postgresql://user:password@host:5432/database + +def get_database_url() -> str: + """Get database URL from environment.""" + url = os.environ.get("LAKEBASE_PG_URL") + if url and url.startswith("postgresql://"): + # Convert to psycopg3 async driver + url = url.replace("postgresql://", "postgresql+psycopg://", 1) + return url + +engine = create_async_engine( + get_database_url(), + pool_size=5, + connect_args={"sslmode": "require"} +) +``` + +### 4. DNS Resolution Workaround (macOS) + +Python's `socket.getaddrinfo()` fails with long hostnames on macOS. Use `dig` as fallback: + +```python +import subprocess +import socket + +def resolve_hostname(hostname: str) -> str: + """Resolve hostname using dig command (macOS workaround).""" + try: + # Try Python's resolver first + return socket.gethostbyname(hostname) + except socket.gaierror: + pass + + # Fallback to dig command + try: + result = subprocess.run( + ["dig", "+short", hostname], + capture_output=True, + text=True, + timeout=5 + ) + ips = result.stdout.strip().split('\n') + for ip in ips: + if ip and not ip.startswith(';'): + return ip + except Exception: + pass + + raise RuntimeError(f"Could not resolve hostname: {hostname}") + +# Use with psycopg +conn_params = { + "host": hostname, # For TLS SNI + "hostaddr": resolve_hostname(hostname), # Actual IP + "dbname": database_name, + "user": username, + "password": token, + "sslmode": "require" +} +conn = psycopg.connect(**conn_params) +``` + +## Environment Variables + +| Variable | Description | Required | +|----------|-------------|----------| +| `LAKEBASE_PG_URL` | Static PostgreSQL URL (local dev) | Either this OR instance/database | +| `LAKEBASE_INSTANCE_NAME` | Lakebase instance name | With DATABASE_NAME | +| `LAKEBASE_DATABASE_NAME` | Database name | With INSTANCE_NAME | +| `LAKEBASE_USERNAME` | Override username | No | +| `LAKEBASE_HOST` | Override host | No | +| `DB_POOL_SIZE` | Connection pool size | No (default: 5) | +| `DB_MAX_OVERFLOW` | Max pool overflow | No (default: 10) | +| `DB_POOL_RECYCLE_INTERVAL` | Pool recycle seconds | No (default: 3600) | + +## Best Practices + +1. **Always use SSL**: Set `sslmode=require` in all connections +2. **Implement token refresh**: Tokens expire after 1 hour; refresh at 50 minutes +3. **Use connection pooling**: Avoid creating new connections per request +4. **Handle DNS issues on macOS**: Use the `hostaddr` workaround if needed +5. **Close connections properly**: Use context managers or explicit cleanup +6. **Log token refresh events**: Helps debug authentication issues diff --git a/.claude/skills/databricks-lakebase-provisioned/reverse-etl.md b/.claude/skills/databricks-lakebase-provisioned/reverse-etl.md new file mode 100644 index 00000000..9bf17bd7 --- /dev/null +++ b/.claude/skills/databricks-lakebase-provisioned/reverse-etl.md @@ -0,0 +1,226 @@ +# Reverse ETL with Lakebase + +## Overview + +Reverse ETL allows you to sync data from Unity Catalog Delta tables into Lakebase Provisioned as PostgreSQL tables. This enables OLTP access patterns on data processed in the Lakehouse. + +## Creating Synced Tables + +### Using Python SDK + +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +# Create a synced table from Unity Catalog +synced_table = w.database.create_synced_table( + instance_name="my-lakebase-instance", + source_table_name="catalog.schema.source_table", + target_table_name="target_table", + sync_mode="FULL", # FULL or INCREMENTAL +) + +print(f"Synced table created: {synced_table.target_table_name}") +``` + +### Using SQL + +```sql +-- Create synced table via SQL +CREATE SYNCED TABLE my_lakebase.target_table +FROM catalog.schema.source_table +USING LAKEBASE INSTANCE 'my-lakebase-instance'; +``` + +### Using CLI + +```bash +databricks database create-synced-table \ + --instance-name my-lakebase-instance \ + --source-table-name catalog.schema.source_table \ + --target-table-name target_table \ + --sync-mode FULL +``` + +## Sync Modes + +### Full Sync + +Complete replacement of target table on each sync: + +```python +synced_table = w.database.create_synced_table( + instance_name="my-lakebase-instance", + source_table_name="catalog.schema.customers", + target_table_name="customers", + sync_mode="FULL" +) +``` + +**Use when:** +- Source table is small-medium size +- Need complete consistency with source +- Incremental changes are complex to track + +### Incremental Sync + +Only sync changed rows (requires change tracking): + +```python +synced_table = w.database.create_synced_table( + instance_name="my-lakebase-instance", + source_table_name="catalog.schema.events", + target_table_name="events", + sync_mode="INCREMENTAL", + incremental_column="updated_at" # Column to track changes +) +``` + +**Use when:** +- Source table is large +- Have reliable change tracking column +- Minimize sync time and resource usage + +## Managing Synced Tables + +### List Synced Tables + +```python +synced_tables = w.database.list_synced_tables( + instance_name="my-lakebase-instance" +) +for table in synced_tables: + print(f"{table.target_table_name}: {table.sync_status}") +``` + +### Trigger Manual Sync + +```python +w.database.sync_table( + instance_name="my-lakebase-instance", + table_name="customers" +) +``` + +### Delete Synced Table + +```python +w.database.delete_synced_table( + instance_name="my-lakebase-instance", + table_name="customers" +) +``` + +## Scheduling Syncs + +### Using Databricks Jobs + +```python +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.jobs import Task, NotebookTask, CronSchedule + +w = WorkspaceClient() + +# Create job to sync tables on schedule +job = w.jobs.create( + name="Lakebase Sync Job", + tasks=[ + Task( + task_key="sync_customers", + notebook_task=NotebookTask( + notebook_path="/Repos/sync/sync_customers" + ) + ) + ], + schedule=CronSchedule( + quartz_cron_expression="0 0 * * * ?", # Every hour + timezone_id="UTC" + ) +) +``` + +### Sync Notebook Example + +```python +# Databricks notebook: sync_customers + +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +# Trigger sync for specific tables +tables_to_sync = ["customers", "orders", "products"] + +for table in tables_to_sync: + try: + w.database.sync_table( + instance_name="my-lakebase-instance", + table_name=table + ) + print(f"Synced: {table}") + except Exception as e: + print(f"Failed to sync {table}: {e}") +``` + +## Use Cases + +### 1. Product Catalog for Web App + +```python +# Sync product data for e-commerce app +w.database.create_synced_table( + instance_name="ecommerce-db", + source_table_name="gold.products.catalog", + target_table_name="products", + sync_mode="FULL" +) + +# Application queries PostgreSQL directly +# with low-latency point lookups +``` + +### 2. User Profiles for Authentication + +```python +# Sync user profiles for auth service +w.database.create_synced_table( + instance_name="auth-db", + source_table_name="gold.users.profiles", + target_table_name="user_profiles", + sync_mode="INCREMENTAL", + incremental_column="last_modified" +) +``` + +### 3. Feature Store for Real-time ML + +```python +# Sync features for online serving +w.database.create_synced_table( + instance_name="feature-store-db", + source_table_name="ml.features.user_features", + target_table_name="user_features", + sync_mode="INCREMENTAL", + incremental_column="computed_at" +) + +# ML model queries features with low latency +``` + +## Best Practices + +1. **Choose appropriate sync mode**: Use FULL for small tables, INCREMENTAL for large tables with change tracking +2. **Schedule during low-traffic periods**: Heavy syncs can impact both source and target +3. **Monitor sync status**: Check for failures and latency +4. **Index target tables**: Create appropriate indexes in PostgreSQL for query patterns +5. **Handle schema changes**: Synced tables need updates when source schema changes + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Sync takes too long** | Switch to INCREMENTAL mode; add indexes on source | +| **Schema mismatch** | Drop and recreate synced table after source schema changes | +| **Sync fails with timeout** | Increase sync timeout; reduce batch size | +| **Target table locked** | Avoid DDL on target during sync operations | diff --git a/.claude/skills/databricks-metric-views/SKILL.md b/.claude/skills/databricks-metric-views/SKILL.md new file mode 100644 index 00000000..d3f5834c --- /dev/null +++ b/.claude/skills/databricks-metric-views/SKILL.md @@ -0,0 +1,229 @@ +--- +name: databricks-metric-views +description: "Unity Catalog metric views: define, create, query, and manage governed business metrics in YAML. Use when building standardized KPIs, revenue metrics, order analytics, or any reusable business metrics that need consistent definitions across teams and tools." +--- + +# Unity Catalog Metric Views + +Define reusable, governed business metrics in YAML that separate measure definitions from dimension groupings for flexible querying. + +## When to Use + +Use this skill when: +- Defining **standardized business metrics** (revenue, order counts, conversion rates) +- Building **KPI layers** shared across dashboards, Genie, and SQL queries +- Creating metrics with **complex aggregations** (ratios, distinct counts, filtered measures) +- Defining **window measures** (moving averages, running totals, period-over-period, YTD) +- Modeling **star or snowflake schemas** with joins in metric definitions +- Enabling **materialization** for pre-computed metric aggregations + +## Prerequisites + +- **Databricks Runtime 17.2+** (for YAML version 1.1) +- SQL warehouse with `CAN USE` permissions +- `SELECT` on source tables, `CREATE TABLE` + `USE SCHEMA` in the target schema + +## Quick Start + +### Create a Metric View + +```sql +CREATE OR REPLACE VIEW catalog.schema.orders_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + comment: "Orders KPIs for sales analysis" + source: catalog.schema.orders + filter: order_date > '2020-01-01' + dimensions: + - name: Order Month + expr: DATE_TRUNC('MONTH', order_date) + comment: "Month of order" + - name: Order Status + expr: CASE + WHEN status = 'O' THEN 'Open' + WHEN status = 'P' THEN 'Processing' + WHEN status = 'F' THEN 'Fulfilled' + END + comment: "Human-readable order status" + measures: + - name: Order Count + expr: COUNT(1) + - name: Total Revenue + expr: SUM(total_price) + comment: "Sum of total price" + - name: Revenue per Customer + expr: SUM(total_price) / COUNT(DISTINCT customer_id) + comment: "Average revenue per unique customer" +$$ +``` + +### Query a Metric View + +All measures must use the `MEASURE()` function. `SELECT *` is NOT supported. + +```sql +SELECT + `Order Month`, + `Order Status`, + MEASURE(`Total Revenue`) AS total_revenue, + MEASURE(`Order Count`) AS order_count +FROM catalog.schema.orders_metrics +WHERE extract(year FROM `Order Month`) = 2024 +GROUP BY ALL +ORDER BY ALL +``` + +## Reference Files + +| Topic | File | Description | +|-------|------|-------------| +| YAML Syntax | [yaml-reference.md](yaml-reference.md) | Complete YAML spec: dimensions, measures, joins, materialization | +| Patterns & Examples | [patterns.md](patterns.md) | Common patterns: star schema, snowflake, filtered measures, window measures, ratios | + +## MCP Tools + +Use the `manage_metric_views` tool for all metric view operations: + +| Action | Description | +|--------|-------------| +| `create` | Create a metric view with dimensions and measures | +| `alter` | Update a metric view's YAML definition | +| `describe` | Get the full definition and metadata | +| `query` | Query measures grouped by dimensions | +| `drop` | Drop a metric view | +| `grant` | Grant SELECT privileges to users/groups | + +### Create via MCP + +```python +manage_metric_views( + action="create", + full_name="catalog.schema.orders_metrics", + source="catalog.schema.orders", + or_replace=True, + comment="Orders KPIs for sales analysis", + filter_expr="order_date > '2020-01-01'", + dimensions=[ + {"name": "Order Month", "expr": "DATE_TRUNC('MONTH', order_date)", "comment": "Month of order"}, + {"name": "Order Status", "expr": "status"}, + ], + measures=[ + {"name": "Order Count", "expr": "COUNT(1)"}, + {"name": "Total Revenue", "expr": "SUM(total_price)", "comment": "Sum of total price"}, + ], +) +``` + +### Query via MCP + +```python +manage_metric_views( + action="query", + full_name="catalog.schema.orders_metrics", + query_measures=["Total Revenue", "Order Count"], + query_dimensions=["Order Month"], + where="extract(year FROM `Order Month`) = 2024", + order_by="ALL", + limit=100, +) +``` + +### Describe via MCP + +```python +manage_metric_views( + action="describe", + full_name="catalog.schema.orders_metrics", +) +``` + +### Grant Access + +```python +manage_metric_views( + action="grant", + full_name="catalog.schema.orders_metrics", + principal="data-consumers", + privileges=["SELECT"], +) +``` + +## YAML Spec Quick Reference + +```yaml +version: 1.1 # Required: "1.1" for DBR 17.2+ +comment: "Description" # Optional: metric view description +source: catalog.schema.table # Required: source table/view +filter: column > value # Optional: global WHERE filter + +dimensions: # Required: at least one + - name: Display Name # Backtick-quoted in queries + expr: sql_expression # Column ref or SQL transformation + comment: "Description" # Optional (v1.1+) + +measures: # Required: at least one + - name: Display Name # Queried via MEASURE(`name`) + expr: AGG_FUNC(column) # Must be an aggregate expression + comment: "Description" # Optional (v1.1+) + +joins: # Optional: star/snowflake schema + - name: dim_table + source: catalog.schema.dim_table + on: source.fk = dim_table.pk + +materialization: # Optional (experimental) + schedule: every 6 hours + mode: relaxed +``` + +## Key Concepts + +### Dimensions vs Measures + +| | Dimensions | Measures | +|---|---|---| +| **Purpose** | Categorize and group data | Aggregate numeric values | +| **Examples** | Region, Date, Status | SUM(revenue), COUNT(orders) | +| **In queries** | Used in SELECT and GROUP BY | Wrapped in `MEASURE()` | +| **SQL expressions** | Any SQL expression | Must use aggregate functions | + +### Why Metric Views vs Standard Views? + +| Feature | Standard Views | Metric Views | +|---------|---------------|--------------| +| Aggregation locked at creation | Yes | No - flexible at query time | +| Safe re-aggregation of ratios | No | Yes | +| Star/snowflake schema joins | Manual | Declarative in YAML | +| Materialization | Separate MV needed | Built-in | +| AI/BI Genie integration | Limited | Native | + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **SELECT * not supported** | Must explicitly list dimensions and use MEASURE() for measures | +| **"Cannot resolve column"** | Dimension/measure names with spaces need backtick quoting | +| **JOIN at query time fails** | Joins must be in the YAML definition, not in the SELECT query | +| **MEASURE() required** | All measure references must be wrapped: `MEASURE(\`name\`)` | +| **DBR version error** | Requires Runtime 17.2+ for YAML v1.1, or 16.4+ for v0.1 | +| **Materialization not working** | Requires serverless compute enabled; currently experimental | + +## Integrations + +Metric views work natively with: +- **AI/BI Dashboards** - Use as datasets for visualizations +- **AI/BI Genie** - Natural language querying of metrics +- **Alerts** - Set threshold-based alerts on measures +- **SQL Editor** - Direct SQL querying with MEASURE() +- **Catalog Explorer UI** - Visual creation and browsing + +## Resources + +- [Metric Views Documentation](https://docs.databricks.com/en/metric-views/) +- [YAML Syntax Reference](https://docs.databricks.com/en/metric-views/data-modeling/syntax) +- [Joins](https://docs.databricks.com/en/metric-views/data-modeling/joins) +- [Window Measures](https://docs.databricks.com/aws/en/metric-views/data-modeling/window-measures) (Experimental) +- [Materialization](https://docs.databricks.com/en/metric-views/materialization) +- [MEASURE() Function](https://docs.databricks.com/en/sql/language-manual/functions/measure) diff --git a/.claude/skills/databricks-metric-views/patterns.md b/.claude/skills/databricks-metric-views/patterns.md new file mode 100644 index 00000000..48c7f9e3 --- /dev/null +++ b/.claude/skills/databricks-metric-views/patterns.md @@ -0,0 +1,651 @@ +# Metric View Patterns & Examples + +Common patterns for creating and querying metric views. + +## Pattern 1: Simple Metrics from a Single Table + +The most basic pattern with direct column dimensions and standard aggregations. + +### Create + +```sql +CREATE OR REPLACE VIEW catalog.schema.product_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + comment: "Product sales metrics" + source: catalog.schema.sales + dimensions: + - name: Product Name + expr: product_name + - name: Sale Date + expr: sale_date + measures: + - name: Units Sold + expr: COUNT(1) + - name: Total Revenue + expr: SUM(price * quantity) + - name: Average Price + expr: AVG(price) +$$ +``` + +### Query + +```sql +-- Revenue by product +SELECT + `Product Name`, + MEASURE(`Total Revenue`) AS revenue, + MEASURE(`Units Sold`) AS units +FROM catalog.schema.product_metrics +GROUP BY ALL +ORDER BY revenue DESC +LIMIT 10 + +-- Monthly trend +SELECT + DATE_TRUNC('MONTH', `Sale Date`) AS month, + MEASURE(`Total Revenue`) AS revenue +FROM catalog.schema.product_metrics +GROUP BY ALL +ORDER BY month +``` + +## Pattern 2: Derived Dimensions with CASE + +Transform raw values into business-friendly categories. + +```sql +CREATE OR REPLACE VIEW catalog.schema.order_kpis +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + source: catalog.schema.orders + dimensions: + - name: Order Month + expr: DATE_TRUNC('MONTH', order_date) + - name: Priority Level + expr: CASE + WHEN priority <= 2 THEN 'High' + WHEN priority <= 4 THEN 'Medium' + ELSE 'Low' + END + comment: "Bucketed priority: High (1-2), Medium (3-4), Low (5)" + - name: Size Category + expr: CASE + WHEN total_amount > 10000 THEN 'Large' + WHEN total_amount > 1000 THEN 'Medium' + ELSE 'Small' + END + measures: + - name: Order Count + expr: COUNT(1) + - name: Total Amount + expr: SUM(total_amount) +$$ +``` + +## Pattern 3: Ratio Measures + +Ratios and per-unit metrics that safely handle re-aggregation. + +```sql +CREATE OR REPLACE VIEW catalog.schema.efficiency_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + comment: "Efficiency and per-unit metrics" + source: catalog.schema.transactions + dimensions: + - name: Department + expr: department_name + - name: Quarter + expr: DATE_TRUNC('QUARTER', transaction_date) + measures: + - name: Total Revenue + expr: SUM(revenue) + - name: Total Cost + expr: SUM(cost) + - name: Profit Margin + expr: (SUM(revenue) - SUM(cost)) / SUM(revenue) + comment: "Profit as percentage of revenue" + - name: Revenue per Employee + expr: SUM(revenue) / COUNT(DISTINCT employee_id) + - name: Average Transaction Size + expr: SUM(revenue) / COUNT(1) +$$ +``` + +## Pattern 4: Filtered Measures (FILTER clause) + +Create measures that only count a subset of rows. + +```sql +CREATE OR REPLACE VIEW catalog.schema.order_status_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + source: catalog.schema.orders + dimensions: + - name: Order Month + expr: DATE_TRUNC('MONTH', order_date) + - name: Region + expr: region + measures: + - name: Total Orders + expr: COUNT(1) + - name: Open Orders + expr: COUNT(1) FILTER (WHERE status = 'OPEN') + - name: Fulfilled Orders + expr: COUNT(1) FILTER (WHERE status = 'FULFILLED') + - name: Open Revenue + expr: SUM(amount) FILTER (WHERE status = 'OPEN') + comment: "Revenue at risk from unfulfilled orders" + - name: Fulfillment Rate + expr: COUNT(1) FILTER (WHERE status = 'FULFILLED') * 1.0 / COUNT(1) + comment: "Percentage of orders fulfilled" +$$ +``` + +### Query filtered measures + +```sql +SELECT + `Order Month`, + MEASURE(`Total Orders`) AS total, + MEASURE(`Open Orders`) AS open_orders, + MEASURE(`Fulfillment Rate`) AS fulfillment_rate +FROM catalog.schema.order_status_metrics +WHERE `Region` = 'EMEA' +GROUP BY ALL +ORDER BY ALL +``` + +## Pattern 5: Star Schema with Joins + +Join a fact table to dimension tables. + +```sql +CREATE OR REPLACE VIEW catalog.schema.sales_analytics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + comment: "Sales analytics with customer and product dimensions" + source: catalog.schema.fact_sales + + joins: + - name: customer + source: catalog.schema.dim_customer + on: source.customer_id = customer.customer_id + - name: product + source: catalog.schema.dim_product + on: source.product_id = product.product_id + - name: store + source: catalog.schema.dim_store + on: source.store_id = store.store_id + + dimensions: + - name: Customer Segment + expr: customer.segment + - name: Product Category + expr: product.category + - name: Store City + expr: store.city + - name: Sale Month + expr: DATE_TRUNC('MONTH', source.sale_date) + + measures: + - name: Total Revenue + expr: SUM(source.amount) + - name: Unique Customers + expr: COUNT(DISTINCT source.customer_id) + - name: Average Basket Size + expr: SUM(source.amount) / COUNT(DISTINCT source.transaction_id) +$$ +``` + +## Pattern 6: Snowflake Schema (Nested Joins) + +Multi-level dimension hierarchies. Requires DBR 17.1+. + +```sql +CREATE OR REPLACE VIEW catalog.schema.geo_sales +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + source: catalog.schema.orders + + joins: + - name: customer + source: catalog.schema.customer + on: source.customer_key = customer.customer_key + joins: + - name: nation + source: catalog.schema.nation + on: customer.nation_key = nation.nation_key + joins: + - name: region + source: catalog.schema.region + on: nation.region_key = region.region_key + + dimensions: + - name: Customer Name + expr: customer.name + - name: Nation + expr: nation.name + - name: Region + expr: region.name + - name: Order Year + expr: EXTRACT(YEAR FROM source.order_date) + + measures: + - name: Total Revenue + expr: SUM(source.total_price) + - name: Order Count + expr: COUNT(1) +$$ +``` + +### Query across hierarchy levels + +```sql +-- Revenue by region (rolls up across nations and customers) +SELECT + `Region`, + MEASURE(`Total Revenue`) AS revenue +FROM catalog.schema.geo_sales +GROUP BY ALL + +-- Revenue by nation within a specific region +SELECT + `Nation`, + MEASURE(`Total Revenue`) AS revenue, + MEASURE(`Order Count`) AS orders +FROM catalog.schema.geo_sales +WHERE `Region` = 'EUROPE' +GROUP BY ALL +ORDER BY revenue DESC +``` + +## Pattern 7: Materialized Metric View + +Pre-compute common aggregations for faster queries. + +```sql +CREATE OR REPLACE VIEW catalog.schema.ecommerce_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + source: catalog.schema.transactions + + dimensions: + - name: Category + expr: product_category + - name: Day + expr: DATE_TRUNC('DAY', transaction_date) + - name: Channel + expr: sales_channel + + measures: + - name: Revenue + expr: SUM(amount) + - name: Transactions + expr: COUNT(1) + - name: Unique Buyers + expr: COUNT(DISTINCT customer_id) + + materialization: + schedule: every 1 hour + mode: relaxed + materialized_views: + - name: daily_category + type: aggregated + dimensions: + - Category + - Day + measures: + - Revenue + - Transactions + - name: full_model + type: unaggregated +$$ +``` + +## Pattern 8: Using samples.tpch for Quick Demos + +The TPC-H sample dataset is available on all Databricks workspaces. + +```sql +CREATE OR REPLACE VIEW catalog.schema.tpch_orders_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + comment: "TPC-H Orders KPIs - demo metric view" + source: samples.tpch.orders + filter: o_orderdate > '1990-01-01' + + dimensions: + - name: Order Month + expr: DATE_TRUNC('MONTH', o_orderdate) + comment: "Month of order" + - name: Order Status + expr: CASE + WHEN o_orderstatus = 'O' THEN 'Open' + WHEN o_orderstatus = 'P' THEN 'Processing' + WHEN o_orderstatus = 'F' THEN 'Fulfilled' + END + comment: "Status: Open, Processing, or Fulfilled" + - name: Order Priority + expr: SPLIT(o_orderpriority, '-')[1] + comment: "Numeric priority 1-5; 1 is highest" + + measures: + - name: Order Count + expr: COUNT(1) + - name: Total Revenue + expr: SUM(o_totalprice) + comment: "Sum of total price" + - name: Revenue per Customer + expr: SUM(o_totalprice) / COUNT(DISTINCT o_custkey) + comment: "Average revenue per distinct customer" + - name: Open Order Revenue + expr: SUM(o_totalprice) FILTER (WHERE o_orderstatus = 'O') + comment: "Potential revenue from open orders" +$$ +``` + +### Demo queries + +```sql +-- Monthly revenue trend +SELECT + `Order Month`, + MEASURE(`Total Revenue`)::BIGINT AS revenue, + MEASURE(`Order Count`) AS orders +FROM catalog.schema.tpch_orders_metrics +WHERE extract(year FROM `Order Month`) = 1995 +GROUP BY ALL +ORDER BY ALL + +-- Revenue by status +SELECT + `Order Status`, + MEASURE(`Total Revenue`)::BIGINT AS revenue, + MEASURE(`Revenue per Customer`)::BIGINT AS rev_per_customer +FROM catalog.schema.tpch_orders_metrics +GROUP BY ALL + +-- Open orders risk assessment +SELECT + `Order Month`, + MEASURE(`Open Order Revenue`)::BIGINT AS at_risk_revenue, + MEASURE(`Total Revenue`)::BIGINT AS total_revenue +FROM catalog.schema.tpch_orders_metrics +WHERE extract(year FROM `Order Month`) >= 1995 +GROUP BY ALL +ORDER BY ALL +``` + +## Pattern 9: Window Measures (Experimental) + +Window measures enable moving averages, running totals, period-over-period changes, and semiadditive measures. Add a `window` block to any measure definition. See [Window Measures Documentation](https://docs.databricks.com/aws/en/metric-views/data-modeling/window-measures). + +### Window Range Values + +| Range | Description | +|-------|-------------| +| `current` | Only rows where the window ordering value equals the current row | +| `cumulative` | All rows up to and including the current row | +| `trailing ` | N units before the current row (**excludes** current) | +| `leading ` | N units after the current row | +| `all` | All rows regardless of ordering | + +### Trailing Window: 7-Day Distinct Customers + +```sql +CREATE OR REPLACE VIEW catalog.schema.customer_activity +WITH METRICS +LANGUAGE YAML +AS $$ + version: 0.1 + source: catalog.schema.orders + filter: order_date > DATE'2024-01-01' + + dimensions: + - name: date + expr: order_date + + measures: + - name: t7d_customers + expr: COUNT(DISTINCT customer_id) + window: + - order: date + range: trailing 7 day + semiadditive: last +$$ +``` + +**Key:** `trailing 7 day` includes the 7 days **before** each date, **excluding** the current date. `semiadditive: last` returns the last value when the `date` dimension is not in the GROUP BY. + +### Running Total (Cumulative) + +```sql +CREATE OR REPLACE VIEW catalog.schema.cumulative_sales +WITH METRICS +LANGUAGE YAML +AS $$ + version: 0.1 + source: catalog.schema.orders + filter: order_date > DATE'2024-01-01' + + dimensions: + - name: date + expr: order_date + + measures: + - name: running_total_sales + expr: SUM(total_price) + window: + - order: date + range: cumulative + semiadditive: last +$$ +``` + +### Period-Over-Period: Day-Over-Day Growth + +Compose window measures using `MEASURE()` references in derived measures. + +```sql +CREATE OR REPLACE VIEW catalog.schema.daily_growth +WITH METRICS +LANGUAGE YAML +AS $$ + version: 0.1 + source: catalog.schema.orders + filter: order_date > DATE'2024-01-01' + + dimensions: + - name: date + expr: order_date + + measures: + - name: previous_day_sales + expr: SUM(total_price) + window: + - order: date + range: trailing 1 day + semiadditive: last + + - name: current_day_sales + expr: SUM(total_price) + window: + - order: date + range: current + semiadditive: last + + - name: day_over_day_growth + expr: (MEASURE(current_day_sales) - MEASURE(previous_day_sales)) / MEASURE(previous_day_sales) * 100 +$$ +``` + +**Key:** The derived `day_over_day_growth` measure uses `MEASURE()` to reference other window measures. It does NOT need its own `window` block. + +### Year-to-Date (Composing Multiple Windows) + +A single measure can have multiple window specs to create period-to-date calculations. + +```sql +CREATE OR REPLACE VIEW catalog.schema.ytd_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 0.1 + source: catalog.schema.orders + filter: order_date > DATE'2023-01-01' + + dimensions: + - name: date + expr: order_date + - name: year + expr: DATE_TRUNC('year', order_date) + + measures: + - name: ytd_sales + expr: SUM(total_price) + window: + - order: date + range: cumulative + semiadditive: last + - order: year + range: current + semiadditive: last +$$ +``` + +**Key:** The first window does a cumulative sum over `date`. The second window restricts scope to the `current` year. Together they produce year-to-date. + +### Semiadditive Measure: Bank Balance + +For measures like balances that should not be summed across time. + +```sql +CREATE OR REPLACE VIEW catalog.schema.account_balances +WITH METRICS +LANGUAGE YAML +AS $$ + version: 0.1 + source: catalog.schema.daily_balances + + dimensions: + - name: date + expr: date + - name: customer + expr: customer_id + + measures: + - name: balance + expr: SUM(balance) + window: + - order: date + range: current + semiadditive: last +$$ +``` + +**Key:** `semiadditive: last` prevents summing across dates (returns the last date's value instead), but the measure **still aggregates across other dimensions** like `customer`. When grouped by date, you get total balance across all customers for that day. When not grouped by date, you get the balance from the most recent date. + +### Query window measures + +Window measures are queried with the same `MEASURE()` syntax: + +```sql +SELECT + date, + MEASURE(t7d_customers) AS trailing_7d_customers, + MEASURE(running_total_sales) AS running_total +FROM catalog.schema.customer_activity +WHERE date >= DATE'2024-06-01' +GROUP BY ALL +ORDER BY ALL +``` + +## MCP Tool Examples + +### Create with joins + +```python +manage_metric_views( + action="create", + full_name="catalog.schema.sales_metrics", + source="catalog.schema.fact_sales", + or_replace=True, + joins=[ + { + "name": "customer", + "source": "catalog.schema.dim_customer", + "on": "source.customer_id = customer.id" + }, + { + "name": "product", + "source": "catalog.schema.dim_product", + "on": "source.product_id = product.id" + } + ], + dimensions=[ + {"name": "Customer Segment", "expr": "customer.segment"}, + {"name": "Product Category", "expr": "product.category"}, + {"name": "Sale Month", "expr": "DATE_TRUNC('MONTH', source.sale_date)"}, + ], + measures=[ + {"name": "Total Revenue", "expr": "SUM(source.amount)"}, + {"name": "Order Count", "expr": "COUNT(1)"}, + {"name": "Unique Customers", "expr": "COUNT(DISTINCT source.customer_id)"}, + ], +) +``` + +### Alter to add a new measure + +```python +manage_metric_views( + action="alter", + full_name="catalog.schema.sales_metrics", + source="catalog.schema.fact_sales", + joins=[ + {"name": "customer", "source": "catalog.schema.dim_customer", "on": "source.customer_id = customer.id"}, + ], + dimensions=[ + {"name": "Customer Segment", "expr": "customer.segment"}, + {"name": "Sale Month", "expr": "DATE_TRUNC('MONTH', source.sale_date)"}, + ], + measures=[ + {"name": "Total Revenue", "expr": "SUM(source.amount)"}, + {"name": "Order Count", "expr": "COUNT(1)"}, + {"name": "Average Order Value", "expr": "AVG(source.amount)"}, # New measure + ], +) +``` + +### Query with filters + +```python +manage_metric_views( + action="query", + full_name="catalog.schema.sales_metrics", + query_measures=["Total Revenue", "Order Count"], + query_dimensions=["Customer Segment", "Sale Month"], + where="`Customer Segment` = 'Enterprise'", + order_by="ALL", + limit=50, +) +``` diff --git a/.claude/skills/databricks-metric-views/yaml-reference.md b/.claude/skills/databricks-metric-views/yaml-reference.md new file mode 100644 index 00000000..2e5973c0 --- /dev/null +++ b/.claude/skills/databricks-metric-views/yaml-reference.md @@ -0,0 +1,338 @@ +# Metric View YAML Reference + +Complete reference for the YAML specification used in Unity Catalog metric views. + +## Top-Level Fields + +| Field | Required | Type | Description | +|-------|----------|------|-------------| +| `version` | No | string | YAML spec version. `"1.1"` for DBR 17.2+, `"0.1"` for DBR 16.4-17.1. Defaults to `1.1`. | +| `source` | Yes | string | Source table, view, or SQL query in three-level namespace format. | +| `comment` | No | string | Description of the metric view (v1.1+). | +| `filter` | No | string | SQL boolean expression applied as a global WHERE clause. | +| `dimensions` | Yes | list | Array of dimension definitions (at least one). | +| `measures` | Yes | list | Array of measure definitions (at least one). | +| `joins` | No | list | Star/snowflake schema join definitions. | +| `materialization` | No | object | Pre-computation configuration (experimental). | + +## Dimensions + +Dimensions define the categorical attributes used to group and filter data. + +```yaml +dimensions: + - name: Region # Display name, backtick-quoted in queries + expr: region_name # Direct column reference + comment: "Sales region" # Optional description (v1.1+) + + - name: Order Month + expr: DATE_TRUNC('MONTH', order_date) # SQL transformation + + - name: Order Year + expr: EXTRACT(YEAR FROM `Order Month`) # Can reference other dimensions + + - name: Customer Type + expr: CASE + WHEN customer_tier = 'A' THEN 'Enterprise' + WHEN customer_tier = 'B' THEN 'Mid-Market' + ELSE 'SMB' + END # Multi-line CASE expressions supported + + - name: Nation + expr: customer.c_name # Reference joined table columns +``` + +### Dimension Rules + +- `name` is required and becomes the column name in queries (backtick-quoted if it has spaces) +- `expr` is required and must be a valid SQL expression +- Can reference source columns, SQL functions, CASE expressions, and other dimensions +- Can reference columns from joined tables using `join_name.column_name` +- Cannot use aggregate functions (those belong in measures) + +## Measures + +Measures define aggregated values computed at query time. + +```yaml +measures: + - name: Total Revenue + expr: SUM(total_price) + comment: "Sum of all order prices" + + - name: Order Count + expr: COUNT(1) + + - name: Average Order Value + expr: AVG(total_price) + + - name: Unique Customers + expr: COUNT(DISTINCT customer_id) + + - name: Revenue per Customer # Ratio measure + expr: SUM(total_price) / COUNT(DISTINCT customer_id) + + - name: Open Order Revenue # Filtered measure + expr: SUM(total_price) FILTER (WHERE status = 'O') + comment: "Revenue from open orders only" + + - name: Open Revenue per Customer # Filtered ratio + expr: SUM(total_price) FILTER (WHERE status = 'O') / COUNT(DISTINCT customer_id) FILTER (WHERE status = 'O') +``` + +### Window Measures (Experimental) + +Add a `window` block to a measure for windowed, cumulative, or semiadditive aggregations. See [Window Measures Documentation](https://docs.databricks.com/aws/en/metric-views/data-modeling/window-measures). + +```yaml +measures: + - name: Running Total + expr: SUM(total_price) + window: + - order: date # Dimension that orders the window + range: cumulative # Window extent (see range values below) + semiadditive: last # How to summarize when order dim is not in GROUP BY + + - name: 7-Day Customers + expr: COUNT(DISTINCT customer_id) + window: + - order: date + range: trailing 7 day # 7 days before current, EXCLUDING current day + semiadditive: last +``` + +**Window range values:** + +| Range | Description | +|-------|-------------| +| `current` | Only rows matching the current ordering value | +| `cumulative` | All rows up to and including the current row | +| `trailing ` | N units before current row (excludes current) | +| `leading ` | N units after current row | +| `all` | All rows | + +**Window spec fields:** + +| Field | Required | Description | +|-------|----------|-------------| +| `order` | Yes | Dimension name that determines window ordering | +| `range` | Yes | Window extent (see values above) | +| `semiadditive` | Yes | `first` or `last` - value to use when order dimension is absent from GROUP BY | + +**Multiple windows** can be composed on a single measure (e.g., for year-to-date): + +```yaml + - name: ytd_sales + expr: SUM(total_price) + window: + - order: date + range: cumulative + semiadditive: last + - order: year + range: current + semiadditive: last +``` + +**Derived measures** can reference window measures using `MEASURE()`: + +```yaml + - name: day_over_day_growth + expr: (MEASURE(current_day_sales) - MEASURE(previous_day_sales)) / MEASURE(previous_day_sales) * 100 +``` + +### Measure Rules + +- `name` is required and queried via `MEASURE(\`name\`)` +- `expr` must contain an aggregate function (SUM, COUNT, AVG, MIN, MAX, etc.) +- Supports `FILTER (WHERE ...)` for conditional aggregation +- Supports ratios of aggregates +- Derived measures can reference other measures via `MEASURE()` (used with window measures) +- Window measures use `version: 0.1` (experimental feature) +- `SELECT *` on metric views is NOT supported; must use `MEASURE()` explicitly + +## Joins + +### Star Schema (Single Level) + +```yaml +source: catalog.schema.fact_orders +joins: + - name: customer + source: catalog.schema.dim_customer + on: source.customer_id = customer.id + + - name: product + source: catalog.schema.dim_product + on: source.product_id = product.id +``` + +### Star Schema with USING + +```yaml +joins: + - name: customer + source: catalog.schema.dim_customer + using: + - customer_id + - region_id +``` + +### Snowflake Schema (Nested Joins, DBR 17.1+) + +```yaml +source: catalog.schema.orders +joins: + - name: customer + source: catalog.schema.customer + on: source.customer_id = customer.id + joins: + - name: nation + source: catalog.schema.nation + on: customer.nation_id = nation.id + joins: + - name: region + source: catalog.schema.region + on: nation.region_id = region.id +``` + +### Join Rules + +- `name` is required and used to reference joined columns: `name.column` +- `source` is the fully qualified table/view name +- Use either `on` (expression) or `using` (column list), not both +- In `on`, reference the fact table as `source` and join tables by their `name` +- Nested `joins` create snowflake schema (requires DBR 17.1+) +- Joined tables cannot include MAP type columns + +## Filter + +A global filter applied to all queries as a WHERE clause. + +```yaml +filter: order_date > '2020-01-01' + +# Multiple conditions +filter: order_date > '2020-01-01' AND status != 'CANCELLED' + +# Using joined columns +filter: customer.active = true +``` + +## Materialization (Experimental) + +Pre-compute aggregations for faster query performance. Uses Lakeflow Spark Declarative Pipelines under the hood. + +```yaml +materialization: + schedule: every 6 hours # Same syntax as MV schedule clause + mode: relaxed # Only "relaxed" supported currently + + materialized_views: + - name: baseline + type: unaggregated # Full unaggregated data model + + - name: revenue_breakdown + type: aggregated # Pre-computed aggregation + dimensions: + - category + - region + measures: + - total_revenue + - order_count + + - name: daily_summary + type: aggregated + dimensions: + - order_date + measures: + - total_revenue +``` + +### Materialization Types + +| Type | Description | When to Use | +|------|-------------|-------------| +| `unaggregated` | Materializes full data model (source + joins + filter) | Expensive source views or many joins | +| `aggregated` | Pre-computes specific dimension/measure combos | Frequently queried combinations | + +### Materialization Requirements + +- Serverless compute must be enabled +- Databricks Runtime 17.2+ +- `TRIGGER ON UPDATE` clause is not supported +- Schedule uses same syntax as materialized view schedules + +### Refresh Materialization + +```python +# Find and refresh the pipeline +from databricks.sdk import WorkspaceClient +w = WorkspaceClient() +pipeline_id = "your-pipeline-id" +w.pipelines.start_update(pipeline_id) +``` + +## Complete Example + +```sql +CREATE OR REPLACE VIEW catalog.schema.sales_metrics +WITH METRICS +LANGUAGE YAML +AS $$ + version: 1.1 + comment: "Comprehensive sales metrics with customer and product dimensions" + source: catalog.schema.fact_sales + filter: sale_date >= '2023-01-01' + + joins: + - name: customer + source: catalog.schema.dim_customer + on: source.customer_id = customer.id + joins: + - name: region + source: catalog.schema.dim_region + on: customer.region_id = region.id + - name: product + source: catalog.schema.dim_product + on: source.product_id = product.id + + dimensions: + - name: Sale Month + expr: DATE_TRUNC('MONTH', sale_date) + comment: "Month of sale" + - name: Customer Name + expr: customer.name + - name: Region + expr: region.name + comment: "Geographic region" + - name: Product Category + expr: product.category + + measures: + - name: Total Revenue + expr: SUM(amount) + comment: "Sum of sale amounts" + - name: Transaction Count + expr: COUNT(1) + - name: Unique Customers + expr: COUNT(DISTINCT customer_id) + - name: Average Transaction + expr: AVG(amount) + - name: Revenue per Customer + expr: SUM(amount) / COUNT(DISTINCT customer_id) + comment: "Average revenue per unique customer" + + materialization: + schedule: every 1 hour + mode: relaxed + materialized_views: + - name: hourly_region + type: aggregated + dimensions: + - Sale Month + - Region + measures: + - Total Revenue + - Transaction Count +$$ +``` diff --git a/.claude/skills/databricks-mlflow-evaluation/SKILL.md b/.claude/skills/databricks-mlflow-evaluation/SKILL.md new file mode 100644 index 00000000..45db5f61 --- /dev/null +++ b/.claude/skills/databricks-mlflow-evaluation/SKILL.md @@ -0,0 +1,148 @@ +--- +name: databricks-mlflow-evaluation +description: "MLflow 3 GenAI agent evaluation. Use when writing mlflow.genai.evaluate() code, creating @scorer functions, using built-in scorers (Guidelines, Correctness, Safety, RetrievalGroundedness), building eval datasets from traces, setting up trace ingestion and production monitoring, aligning judges with MemAlign from domain expert feedback, or running optimize_prompts() with GEPA for automated prompt improvement." +--- + +# MLflow 3 GenAI Evaluation + +## Before Writing Any Code + +1. **Read GOTCHAS.md** - 15+ common mistakes that cause failures +2. **Read CRITICAL-interfaces.md** - Exact API signatures and data schemas + +## End-to-End Workflows + +Follow these workflows based on your goal. Each step indicates which reference files to read. + +### Workflow 1: First-Time Evaluation Setup + +For users new to MLflow GenAI evaluation or setting up evaluation for a new agent. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Understand what to evaluate | `user-journeys.md` (Journey 0: Strategy) | +| 2 | Learn API patterns | `GOTCHAS.md` + `CRITICAL-interfaces.md` | +| 3 | Build initial dataset | `patterns-datasets.md` (Patterns 1-4) | +| 4 | Choose/create scorers | `patterns-scorers.md` + `CRITICAL-interfaces.md` (built-in list) | +| 5 | Run evaluation | `patterns-evaluation.md` (Patterns 1-3) | + +### Workflow 2: Production Trace -> Evaluation Dataset + +For building evaluation datasets from production traces. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Search and filter traces | `patterns-trace-analysis.md` (MCP tools section) | +| 2 | Analyze trace quality | `patterns-trace-analysis.md` (Patterns 1-7) | +| 3 | Tag traces for inclusion | `patterns-datasets.md` (Patterns 16-17) | +| 4 | Build dataset from traces | `patterns-datasets.md` (Patterns 6-7) | +| 5 | Add expectations/ground truth | `patterns-datasets.md` (Pattern 2) | + +### Workflow 3: Performance Optimization + +For debugging slow or expensive agent execution. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Profile latency by span | `patterns-trace-analysis.md` (Patterns 4-6) | +| 2 | Analyze token usage | `patterns-trace-analysis.md` (Pattern 9) | +| 3 | Detect context issues | `patterns-context-optimization.md` (Section 5) | +| 4 | Apply optimizations | `patterns-context-optimization.md` (Sections 1-4, 6) | +| 5 | Re-evaluate to measure impact | `patterns-evaluation.md` (Pattern 6-7) | + +### Workflow 4: Regression Detection + +For comparing agent versions and finding regressions. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Establish baseline | `patterns-evaluation.md` (Pattern 4: named runs) | +| 2 | Run current version | `patterns-evaluation.md` (Pattern 1) | +| 3 | Compare metrics | `patterns-evaluation.md` (Patterns 6-7) | +| 4 | Analyze failing traces | `patterns-trace-analysis.md` (Pattern 7) | +| 5 | Debug specific failures | `patterns-trace-analysis.md` (Patterns 8-9) | + +### Workflow 5: Custom Scorer Development + +For creating project-specific evaluation metrics. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Understand scorer interface | `CRITICAL-interfaces.md` (Scorer section) | +| 2 | Choose scorer pattern | `patterns-scorers.md` (Patterns 4-11) | +| 3 | For multi-agent scorers | `patterns-scorers.md` (Patterns 13-16) | +| 4 | Test with evaluation | `patterns-evaluation.md` (Pattern 1) | + +### Workflow 6: Unity Catalog Trace Ingestion & Production Monitoring + +For storing traces in Unity Catalog, instrumenting applications, and enabling continuous production monitoring. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Link UC schema to experiment | `patterns-trace-ingestion.md` (Patterns 1-2) | +| 2 | Set trace destination | `patterns-trace-ingestion.md` (Patterns 3-4) | +| 3 | Instrument your application | `patterns-trace-ingestion.md` (Patterns 5-8) | +| 4 | Configure trace sources (Apps/Serving/OTEL) | `patterns-trace-ingestion.md` (Patterns 9-11) | +| 5 | Enable production monitoring | `patterns-trace-ingestion.md` (Patterns 12-13) | +| 6 | Query and analyze UC traces | `patterns-trace-ingestion.md` (Pattern 14) | + +### Workflow 7: Judge Alignment with MemAlign + +For aligning an LLM judge to match domain expert preferences. A well-aligned judge improves every downstream use: evaluation accuracy, production monitoring signal, and prompt optimization quality. This workflow is valuable on its own, independent of prompt optimization. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Design base judge with `make_judge` (any feedback type) | `patterns-judge-alignment.md` (Pattern 1) | +| 2 | Run evaluate(), tag successful traces | `patterns-judge-alignment.md` (Pattern 2) | +| 3 | Build UC dataset + create SME labeling session | `patterns-judge-alignment.md` (Pattern 3) | +| 4 | Align judge with MemAlign after labeling completes | `patterns-judge-alignment.md` (Pattern 4) | +| 5 | Register aligned judge to experiment | `patterns-judge-alignment.md` (Pattern 5) | +| 6 | Re-evaluate with aligned judge (baseline) | `patterns-judge-alignment.md` (Pattern 6) | + +### Workflow 8: Automated Prompt Optimization with GEPA + +For automatically improving a registered system prompt using `optimize_prompts()`. Works with any scorer, but paired with an aligned judge (Workflow 7) gives the most domain-accurate signal. For the full end-to-end loop combining alignment and optimization, see `user-journeys.md` Journey 10. + +| Step | Action | Reference Files | +|------|--------|-----------------| +| 1 | Build optimization dataset (inputs + expectations) | `patterns-prompt-optimization.md` (Pattern 1) | +| 2 | Run optimize_prompts() with GEPA + scorer | `patterns-prompt-optimization.md` (Pattern 2) | +| 3 | Register new version, promote conditionally | `patterns-prompt-optimization.md` (Pattern 3) | + +## Reference Files Quick Lookup + +| Reference | Purpose | When to Read | +|-----------|---------|--------------| +| `GOTCHAS.md` | Common mistakes | **Always read first** before writing code | +| `CRITICAL-interfaces.md` | API signatures, schemas | When writing any evaluation code | +| `patterns-evaluation.md` | Running evals, comparing | When executing evaluations | +| `patterns-scorers.md` | Custom scorer creation | When built-in scorers aren't enough | +| `patterns-datasets.md` | Dataset building | When preparing evaluation data | +| `patterns-trace-analysis.md` | Trace debugging | When analyzing agent behavior | +| `patterns-context-optimization.md` | Token/latency fixes | When agent is slow or expensive | +| `patterns-trace-ingestion.md` | UC trace setup, monitoring | When setting up trace storage or production monitoring | +| `patterns-judge-alignment.md` | MemAlign judge alignment, labeling sessions, SME feedback | When aligning judges to domain expert preferences | +| `patterns-prompt-optimization.md` | GEPA optimization: build dataset, optimize_prompts(), promote | When running automated prompt improvement | +| `user-journeys.md` | High-level workflows, full domain-expert optimization loop | When starting a new evaluation project or running the full align + optimize cycle | + +## Critical API Facts + +- **Use:** `mlflow.genai.evaluate()` (NOT `mlflow.evaluate()`) +- **Data format:** `{"inputs": {"query": "..."}}` (nested structure required) +- **predict_fn:** Receives `**unpacked kwargs` (not a dict) +- **MemAlign:** Scorer-agnostic (works with any `feedback_value_type` -- float, bool, categorical); token-heavy on the embedding model so set `embedding_model` explicitly +- **Label schema name matching:** The label schema `name` in the labeling session MUST match the judge `name` used in `evaluate()` for `align()` to pair scores +- **Aligned judge scores:** May be lower than unaligned judge scores -- this is expected and means the judge is now more accurate, not that the agent regressed +- **GEPA optimization dataset:** Must have both `inputs` AND `expectations` per record (different from eval dataset) +- **Episodic memory:** Lazily loaded -- `get_scorer()` results won't show episodic memory on print until the judge is first used +- **optimize_prompts:** Requires MLflow >= 3.5.0 + +See `GOTCHAS.md` for complete list. + +## Related Skills + +- **[databricks-docs](../databricks-docs/SKILL.md)** - General Databricks documentation reference +- **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - Deploying models and agents to serving endpoints +- **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Building agents that can be evaluated with this skill +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - SDK patterns used alongside MLflow APIs +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Unity Catalog tables for managed evaluation datasets diff --git a/.claude/skills/mlflow-evaluation/references/CRITICAL-interfaces.md b/.claude/skills/databricks-mlflow-evaluation/references/CRITICAL-interfaces.md similarity index 87% rename from .claude/skills/mlflow-evaluation/references/CRITICAL-interfaces.md rename to .claude/skills/databricks-mlflow-evaluation/references/CRITICAL-interfaces.md index d1b4a1b1..30babcea 100644 --- a/.claude/skills/mlflow-evaluation/references/CRITICAL-interfaces.md +++ b/.claude/skills/databricks-mlflow-evaluation/references/CRITICAL-interfaces.md @@ -12,6 +12,7 @@ - [Judges API (Low-level)](#judges-api-low-level) - [Trace APIs](#trace-apis) - [Evaluation Datasets (MLflow-managed)](#evaluation-datasets-mlflow-managed) +- [Trace Ingestion in Unity Catalog](#trace-ingestion-in-unity-catalog) - [Production Monitoring](#production-monitoring) - [Key Constants](#key-constants) - [Installation](#installation) @@ -385,8 +386,68 @@ results = mlflow.genai.evaluate( --- +## Trace Ingestion in Unity Catalog + +**Version**: MLflow 3.9.0+ (`mlflow[databricks]>=3.9.0`) + +### Setup - Link UC Schema to Experiment +```python +import os +import mlflow +from mlflow.entities import UCSchemaLocation +from mlflow.tracing.enablement import set_experiment_trace_location + +mlflow.set_tracking_uri("databricks") +os.environ["MLFLOW_TRACING_SQL_WAREHOUSE_ID"] = "" + +experiment_id = mlflow.create_experiment(name="/Shared/my-traces") + +set_experiment_trace_location( + location=UCSchemaLocation( + catalog_name="", + schema_name="" + ), + experiment_id=experiment_id, +) +# Creates: mlflow_experiment_trace_otel_logs, _metrics, _spans +``` + +### Set Trace Destination +```python +# Option A: Python API +from mlflow.entities import UCSchemaLocation +mlflow.tracing.set_destination( + destination=UCSchemaLocation( + catalog_name="", + schema_name="", + ) +) + +# Option B: Environment variable +os.environ["MLFLOW_TRACING_DESTINATION"] = "." +``` + +### Permissions Required +- `USE_CATALOG` on catalog +- `USE_SCHEMA` on schema +- `MODIFY` and `SELECT` on each `mlflow_experiment_trace_*` table +- **CRITICAL**: `ALL_PRIVILEGES` is NOT sufficient + +--- + ## Production Monitoring +### Configure Monitoring SQL Warehouse +```python +from mlflow.tracing import set_databricks_monitoring_sql_warehouse_id + +set_databricks_monitoring_sql_warehouse_id( + warehouse_id="", + experiment_id="" # Optional +) +# Alternative: os.environ["MLFLOW_TRACING_SQL_WAREHOUSE_ID"] = "" +``` + ### Register and Start Scorer ```python from mlflow.genai.scorers import Safety, Guidelines, ScorerSamplingConfig diff --git a/.claude/skills/mlflow-evaluation/references/GOTCHAS.md b/.claude/skills/databricks-mlflow-evaluation/references/GOTCHAS.md similarity index 54% rename from .claude/skills/mlflow-evaluation/references/GOTCHAS.md rename to .claude/skills/databricks-mlflow-evaluation/references/GOTCHAS.md index fc40d97b..4e468035 100644 --- a/.claude/skills/mlflow-evaluation/references/GOTCHAS.md +++ b/.claude/skills/databricks-mlflow-evaluation/references/GOTCHAS.md @@ -23,6 +23,15 @@ - [Wrong Production Monitoring Setup](#-wrong-production-monitoring-setup) - [Wrong Custom Judge Model Format](#-wrong-custom-judge-model-format) - [Wrong Aggregation Values](#-wrong-aggregation-values) +- [Wrong Trace Ingestion Setup](#-wrong-trace-ingestion-setup) +- [Wrong Trace Destination Format](#-wrong-trace-destination-format) +- [Wrong MLflow Version for Trace Ingestion](#-wrong-mlflow-version-for-trace-ingestion) +- [Wrong Linking UC Schema Without SQL Warehouse](#-wrong-linking-uc-schema-without-sql-warehouse) +- [Wrong Label Schema Name — Alignment Will Fail](#-wrong-label-schema-name--alignment-will-fail) +- [Wrong Aligned Judge Score Interpretation](#-wrong-aligned-judge-score-interpretation) +- [Wrong MemAlign Embedding Model — Token Costs](#-wrong-memalign-embedding-model--token-costs) +- [Wrong MemAlign Episodic Memory — Lazy Loading](#-wrong-memalign-episodic-memory--lazy-loading) +- [Wrong GEPA Optimization Dataset — Missing expectations](#-wrong-gepa-optimization-dataset--missing-expectations) - [Summary Checklist](#summary-checklist) --- @@ -530,6 +539,253 @@ def my_scorer(outputs) -> float: --- +## ❌ WRONG Trace Ingestion Setup + +### WRONG: Using ALL_PRIVILEGES instead of explicit grants +```sql +-- ❌ WRONG - ALL_PRIVILEGES does NOT include required permissions +GRANT ALL_PRIVILEGES ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_spans + TO `user@company.com`; +``` + +### ✅ CORRECT: Grant explicit MODIFY and SELECT +```sql +-- ✅ CORRECT - Explicit MODIFY and SELECT required +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_spans + TO `user@company.com`; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_logs + TO `user@company.com`; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_metrics + TO `user@company.com`; +``` + +--- + +## ❌ WRONG Trace Destination Format + +### WRONG: Wrong format for environment variable +```python +# ❌ WRONG - Missing schema or wrong separator +os.environ["MLFLOW_TRACING_DESTINATION"] = "my_catalog" +os.environ["MLFLOW_TRACING_DESTINATION"] = "my_catalog/my_schema" +``` + +### ✅ CORRECT: Use catalog.schema format +```python +# ✅ CORRECT - Dot-separated catalog.schema +os.environ["MLFLOW_TRACING_DESTINATION"] = "my_catalog.my_schema" +``` + +--- + +## ❌ WRONG MLflow Version for Trace Ingestion + +### WRONG: Using MLflow < 3.9.0 for UC trace ingestion +```bash +# ❌ WRONG - Trace ingestion requires 3.9.0+ +pip install mlflow[databricks]>=3.1.0 +``` + +### ✅ CORRECT: Use MLflow 3.9.0+ for UC traces +```bash +# ✅ CORRECT +pip install "mlflow[databricks]>=3.9.0" --upgrade --force-reinstall +``` + +--- + +## ❌ WRONG Linking UC Schema Without SQL Warehouse + +### WRONG: Missing SQL warehouse configuration +```python +# ❌ WRONG - No SQL warehouse configured +mlflow.set_tracking_uri("databricks") +# Missing: os.environ["MLFLOW_TRACING_SQL_WAREHOUSE_ID"] = "..." +set_experiment_trace_location(location=UCSchemaLocation(...), ...) +``` + +### ✅ CORRECT: Set SQL warehouse before linking +```python +# ✅ CORRECT - Set warehouse ID first +mlflow.set_tracking_uri("databricks") +os.environ["MLFLOW_TRACING_SQL_WAREHOUSE_ID"] = "" +set_experiment_trace_location(location=UCSchemaLocation(...), ...) +``` + +--- + +## ❌ WRONG Label Schema Name — Alignment Will Fail + +### WRONG: Label schema name does not match the judge name used in evaluate() +```python +# ❌ WRONG - Judge name and label schema name don't match +# Judge is registered as "domain_quality_base" in evaluate() +domain_quality_judge = make_judge(name="domain_quality_base", ...) +registered_base_judge = domain_quality_judge.register(experiment_id=EXPERIMENT_ID) + +# But label schema uses a different name +feedback_schema = label_schemas.create_label_schema( + name="domain_quality_rating", # ❌ Does not match judge name + type="feedback", + ... +) +# align() will not be able to pair SME feedback with LLM judge scores +``` + +### ✅ CORRECT: Label schema name matches the judge name exactly +```python +# ✅ CORRECT - Judge name and label schema name are identical +JUDGE_NAME = "domain_quality_base" + +domain_quality_judge = make_judge(name=JUDGE_NAME, ...) +registered_base_judge = domain_quality_judge.register(experiment_id=EXPERIMENT_ID) + +feedback_schema = label_schemas.create_label_schema( + name=JUDGE_NAME, # ✅ Matches judge name exactly + type="feedback", + ... +) +``` + +**Why?** The `align()` function pairs SME feedback with LLM judge scores by matching the label schema name to the judge name on the same traces. If the names differ, `align()` cannot find the corresponding score pairs and alignment will fail or produce incorrect results. + +--- + +## ❌ WRONG Aligned Judge Score Interpretation + +### WRONG: Assuming a lower aligned judge score means the agent got worse +```python +# ❌ WRONG interpretation - panicking because aligned judge gives lower scores +# Unaligned judge: 4.2/5.0 average +# Aligned judge: 3.1/5.0 average +# "The agent regressed!" — No, the judge got more accurate. +``` + +### ✅ CORRECT: Understanding that a lower aligned score reflects more accurate evaluation +```python +# ✅ CORRECT interpretation +# The aligned judge now evaluates with domain-expert standards rather than generic best practices. +# A lower score from a more accurate judge is a better signal than an inflated score from +# a judge that doesn't understand your domain. The unaligned judge was underspecified. +# Use optimize_prompts() with the aligned judge to improve the agent against this standard. +``` + +**Why?** An unaligned judge evaluates against generic best practices and often gives inflated scores. Once aligned with SME feedback, the judge applies domain-specific criteria that are harder to satisfy. The lower score is not a regression in agent quality; it is a more honest assessment. The optimization phase (`optimize_prompts()`) will then improve the agent against this more accurate standard. + +--- + +## ❌ WRONG MemAlign Embedding Model — Token Costs + +### WRONG: Using the default embedding model without awareness of cost +```python +# ❌ COSTLY - Default embedding model may be expensive for large trace sets +optimizer = MemAlignOptimizer( + reflection_lm=REFLECTION_MODEL, + retrieval_k=5, + # No embedding_model specified → defaults to "openai/text-embedding-3-small" +) +``` + +### ✅ CORRECT: Use a Databricks-hosted embedding model or size your trace set accordingly +```python +# ✅ CORRECT - Use a hosted model to control costs; scope trace set to labeled traces only +optimizer = MemAlignOptimizer( + reflection_lm=REFLECTION_MODEL, + retrieval_k=5, + embedding_model="databricks:/databricks-gte-large-en", +) + +# ✅ ALSO CORRECT - Filter to only labeled/tagged traces, not all experiment traces +traces = mlflow.search_traces( + locations=[EXPERIMENT_ID], + filter_string="tag.eval = 'complete'", # Scope to relevant traces only + return_type="list", +) +aligned_judge = base_judge.align(traces=traces, optimizer=optimizer) +``` + +**Why?** MemAlign embeds every trace for retrieval (`retrieval_k` nearest neighbors per evaluation). Large trace sets with an expensive embedding model multiply quickly. Databricks-hosted models (`databricks:/databricks-gte-large-en`) keep costs on-platform. + +--- + +## ❌ WRONG MemAlign Episodic Memory — Lazy Loading + +### WRONG: Expecting episodic memory to be populated immediately after get_scorer() +```python +# ❌ WRONG - Episodic memory appears empty, looks like alignment didn't work +retrieved_judge = get_scorer(name="domain_quality_base", experiment_id=EXPERIMENT_ID) +print(retrieved_judge._episodic_memory) # Prints: [] — misleading! +print(retrieved_judge._semantic_memory) # Prints: [] — also empty! +``` + +### ✅ CORRECT: Episodic memory is lazily loaded — use the judge first, then inspect +```python +# ✅ CORRECT - Semantic guidelines ARE loaded; episodic memory loads on first use +retrieved_judge = get_scorer(name="domain_quality_base", experiment_id=EXPERIMENT_ID) + +# The instructions field already contains the distilled guidelines — inspect this instead +print(retrieved_judge.instructions) # ✅ Shows full aligned instructions with guidelines + +# To verify episodic memory, run the judge on a sample first, then inspect +# Memory loads lazily when the judge retrieves similar examples during scoring +``` + +**Why?** MemAlign's episodic memory (stored examples) is loaded on-demand when the judge needs to retrieve similar examples at scoring time. The `_episodic_memory` list is empty on deserialization. The aligned `instructions` field (which includes distilled semantic guidelines) is the reliable thing to inspect after `get_scorer()`. + +--- + +## ❌ WRONG GEPA Optimization Dataset — Missing expectations + +### WRONG: Using eval-style dataset (inputs only) for optimize_prompts() +```python +# ❌ WRONG - GEPA requires expectations; optimization will fail or produce poor results +optimization_dataset = [ + {"inputs": {"input": [{"role": "user", "content": "How does the offense attack the blitz?"}]}}, + {"inputs": {"input": [{"role": "user", "content": "What are 3rd down tendencies?"}]}}, +] + +result = mlflow.genai.optimize_prompts( + predict_fn=predict_fn, + train_data=optimization_dataset, # ❌ Missing expectations + prompt_uris=[prompt.uri], + optimizer=GepaPromptOptimizer(...), + scorers=[aligned_judge], +) +``` + +### ✅ CORRECT: Include expectations in every optimization dataset record +```python +# ✅ CORRECT - Each record must have both inputs AND expectations +optimization_dataset = [ + { + "inputs": { + "input": [{"role": "user", "content": "How does the offense attack the blitz?"}] + }, + "expectations": { + "expected_response": ( + "The agent should analyze blitz performance metrics, compare success " + "rates across pressure packages, and provide concrete tactical recommendations." + ) + } + }, + { + "inputs": { + "input": [{"role": "user", "content": "What are 3rd down tendencies?"}] + }, + "expectations": { + "expected_response": ( + "The agent should call the appropriate tool with down=3 parameters, " + "summarize the play distribution, and give defensive recommendations." + ) + } + }, +] +``` + +**Why?** GEPA uses the `expectations` field during reflection — it compares the agent's output against the expected behavior to generate targeted prompt improvement suggestions. Without `expectations`, GEPA cannot reason about *why* the current prompt is underperforming. This is the most common cause of poor optimization results. + +--- + ## Summary Checklist Before running evaluation, verify: @@ -545,3 +801,14 @@ Before running evaluation, verify: - [ ] Production scorers have inline imports - [ ] Multiple Feedbacks have unique names - [ ] Aggregations use valid names: min, max, mean, median, variance, p90 +- [ ] UC trace ingestion uses `mlflow[databricks]>=3.9.0` +- [ ] UC tables have explicit MODIFY + SELECT grants (not ALL_PRIVILEGES) +- [ ] `MLFLOW_TRACING_SQL_WAREHOUSE_ID` set before linking UC schema +- [ ] `MLFLOW_TRACING_DESTINATION` uses `catalog.schema` format (dot-separated) +- [ ] Production monitoring scorers are both registered AND started +- [ ] MemAlign `embedding_model` can be explicitly set (don't rely on default for large trace sets) +- [ ] After `get_scorer()` for a MemAlign judge, inspect `.instructions` not `._episodic_memory` as episodic memory is lazily loaded +- [ ] GEPA `train_data` has both `inputs` AND `expectations` per record +- [ ] Label schema `name` matches the judge `name` used in `evaluate()` (required for `align()` to pair scores) +- [ ] Aligned judge scores may be lower than unaligned — this is expected if the judge is now more accurate +- [ ] MemAlign is scorer-agnostic (works with any `feedback_value_type` — float, bool, categorical) diff --git a/.claude/skills/mlflow-evaluation/references/patterns-context-optimization.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-context-optimization.md similarity index 100% rename from .claude/skills/mlflow-evaluation/references/patterns-context-optimization.md rename to .claude/skills/databricks-mlflow-evaluation/references/patterns-context-optimization.md diff --git a/.claude/skills/mlflow-evaluation/references/patterns-datasets.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-datasets.md similarity index 100% rename from .claude/skills/mlflow-evaluation/references/patterns-datasets.md rename to .claude/skills/databricks-mlflow-evaluation/references/patterns-datasets.md diff --git a/.claude/skills/mlflow-evaluation/references/patterns-evaluation.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-evaluation.md similarity index 100% rename from .claude/skills/mlflow-evaluation/references/patterns-evaluation.md rename to .claude/skills/databricks-mlflow-evaluation/references/patterns-evaluation.md diff --git a/.claude/skills/databricks-mlflow-evaluation/references/patterns-judge-alignment.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-judge-alignment.md new file mode 100644 index 00000000..c59989a9 --- /dev/null +++ b/.claude/skills/databricks-mlflow-evaluation/references/patterns-judge-alignment.md @@ -0,0 +1,316 @@ +# MLflow 3 Judge Alignment with MemAlign + +Patterns for aligning LLM judges to domain expert preferences using MemAlign. An aligned judge is more accurate for evaluation runs, more meaningful for production monitoring, and a better guide for prompt optimization — but each of these uses is independent. + +**Read `GOTCHAS.md` before implementing — especially the MemAlign sections.** + +--- + +## When to Use Judge Alignment + +Align a judge when: +- Built-in scorers don't capture domain-specific quality (e.g., "good" means expert-level tactical analysis) +- LLM judges disagree with human raters on the same examples +- You have domain experts who can rate a sample of agent outputs +- You want production monitoring that reflects actual expert standards + +You do NOT need prompt optimization to benefit from aligned judges — a more accurate judge improves every evaluation run and monitoring setup you do afterward. + +--- + +## Pattern 1: Design and Register the Base Judge + +MemAlign is scorer-agnostic and works with any `feedback_value_type` (float, boolean, categorical). This example uses a Likert scale (1-5 float), but you can use whatever scoring scheme fits your domain. + +```python +import mlflow +from mlflow.genai.judges import make_judge +from mlflow.genai import evaluate + +mlflow.set_experiment(experiment_id=EXPERIMENT_ID) + +# Define base judge using make_judge -- MemAlign works with any feedback type +# This example uses a Likert scale (1-5 float), but boolean or categorical also work +domain_quality_judge = make_judge( + name="domain_quality_base", + instructions=( + "Evaluate if the response in {{ outputs }} appropriately analyzes the available data " + "and provides an actionable recommendation to the question in {{ inputs }}. " + "The response should be accurate, contextually relevant, and give a strategic advantage " + "to the person making the request. " + "Your grading criteria: " + " 1: Completely unacceptable. Incorrect data interpretation or no recommendations. " + " 2: Mostly unacceptable. Irrelevant or spurious feedback or weak recommendations with minimal strategic advantage. " + " 3: Somewhat acceptable. Relevant feedback provided with some strategic advantage. " + " 4: Mostly acceptable. Relevant feedback provided with strong strategic advantage. " + " 5: Completely acceptable. Relevant feedback provided with excellent strategic advantage." + ), + feedback_value_type=float, # Example uses a Likert scale; MemAlign works with any feedback type + model=JUDGE_MODEL, +) + +# Register to experiment — creates the persistent record used by align() +registered_base_judge = domain_quality_judge.register(experiment_id=EXPERIMENT_ID) +print(f"Registered base judge: {registered_base_judge.name}") +``` + +--- + +## Pattern 2: Run Evaluation and Tag Traces + +Run evaluation to generate a set of traces that domain experts will review. Tag traces that were **successfully evaluated** in this `evaluate()` job (i.e., the agent produced a response and the judge scored it without errors). + +```python +from mlflow.genai import evaluate + +# Eval dataset: inputs only (no expectations needed at this stage) +eval_data = [ + {"inputs": {"input": [{"role": "user", "content": question}]}} + for question in example_questions +] + +results = evaluate( + data=eval_data, + predict_fn=lambda input: AGENT.predict({"input": input}), + scorers=[domain_quality_judge], +) + +# Tag traces that were successfully evaluated in this evaluate() job +# "OK" state means the agent responded AND the judge scored it without errors +ok_trace_ids = results.result_df.loc[results.result_df["state"] == "OK", "trace_id"] +for trace_id in ok_trace_ids: + mlflow.set_trace_tag(trace_id=trace_id, key="eval", value="complete") + +print(f"Tagged {len(ok_trace_ids)} successfully evaluated traces for labeling") +``` + +--- + +## Pattern 3: Build Eval Dataset and Create Labeling Session + +Persist traces to a UC dataset and assign them to domain experts for review. + +**CRITICAL: The label schema `name` MUST match the judge `name` used in the `evaluate()` job.** This is how `align()` pairs SME feedback with the corresponding LLM judge scores on the same traces. If these names do not match, alignment will fail or produce incorrect results. + +```python +from mlflow.genai.datasets import create_dataset, get_dataset +from mlflow.genai import create_labeling_session, get_review_app +from mlflow.genai import label_schemas + +# Build persistent dataset from tagged traces +try: + eval_dataset = get_dataset(name=DATASET_NAME) +except Exception: + eval_dataset = create_dataset(name=DATASET_NAME) + +tagged_traces = mlflow.search_traces( + locations=[EXPERIMENT_ID], + filter_string="tag.eval = 'complete'", + return_type="pandas", +) +# merge_records() expects 'inputs' and 'outputs' column names +if "inputs" not in tagged_traces.columns and "request" in tagged_traces.columns: + tagged_traces = tagged_traces.rename(columns={"request": "inputs"}) +if "outputs" not in tagged_traces.columns and "response" in tagged_traces.columns: + tagged_traces = tagged_traces.rename(columns={"response": "outputs"}) + +eval_dataset = eval_dataset.merge_records(tagged_traces) + +# CRITICAL: The label schema name MUST match the judge name used in evaluate() +# This is how align() pairs SME feedback with LLM judge scores on the same traces +LABEL_SCHEMA_NAME = "domain_quality_base" # Must match the judge name exactly + +feedback_schema = label_schemas.create_label_schema( + name=LABEL_SCHEMA_NAME, # Must match judge name from Pattern 1 + type="feedback", + title=LABEL_SCHEMA_NAME, + input=label_schemas.InputNumeric(min_value=1.0, max_value=5.0), + instruction=( + "Evaluate if the response appropriately analyzes the available data and provides " + "an actionable recommendation for the question. The response should be accurate, " + "contextually relevant, and give a strategic advantage to the person making the request. " + "\n\n Your grading criteria should be: " + "\n 1: Completely unacceptable. Incorrect data interpretation or no recommendations." + "\n 2: Mostly unacceptable. Irrelevant or spurious feedback or weak recommendations with minimal strategic advantage." + "\n 3: Somewhat acceptable. Relevant feedback provided with some strategic advantage." + "\n 4: Mostly acceptable. Relevant feedback provided with strong strategic advantage." + "\n 5: Completely acceptable. Relevant feedback provided with excellent strategic advantage." + ), + enable_comment=True, # Allow SMEs to leave free-text rationale (used by MemAlign) + overwrite=True, +) + +# Optional: add a deployed agent to the Review App so SMEs can ask new questions +review_app = get_review_app(experiment_id=EXPERIMENT_ID) +review_app = review_app.add_agent( + agent_name=MODEL_NAME, + model_serving_endpoint=AGENT_ENDPOINT_NAME, + overwrite=True, +) + +# Create labeling session and attach the dataset +labeling_session = create_labeling_session( + name=f"{LABELING_SESSION_NAME}_sme", + assigned_users=ASSIGNED_USERS, + label_schemas=[LABEL_SCHEMA_NAME], # Must match judge name +) +labeling_session = labeling_session.add_dataset(dataset_name=DATASET_NAME) + +print(f"Share with domain experts: {labeling_session.url}") +# Domain experts open this URL and rate each response using the 1-5 scale +``` + +--- + +## Pattern 4: Align Judge with MemAlign (Recommended) + +After SMEs complete labeling, distill their feedback patterns into the judge's instructions. + +Judge alignment supports multiple optimizers (e.g., SIMBA, custom optimizers), but this example uses **MemAlign**, which is the recommended approach. MemAlign is the fastest alignment method (seconds vs. minutes for alternatives), the most cost-effective, and supports **memory scaling** where quality continues to improve as feedback accumulates without re-optimization. + +```python +from mlflow.genai.judges.optimizers import MemAlignOptimizer +from mlflow.genai.scorers import get_scorer + +# Fetch the tagged traces (which now have SME labels attached) +traces_for_alignment = mlflow.search_traces( + locations=[EXPERIMENT_ID], + filter_string="tag.eval = 'complete'", + return_type="list", # align() requires list format +) +print(f"Aligning on {len(traces_for_alignment)} traces") + +# Configure MemAlign optimizer +# Other optimizers are available (e.g., SIMBA), but MemAlign is recommended for its +# speed, cost efficiency, and ability to improve continuously as feedback accumulates +optimizer = MemAlignOptimizer( + reflection_lm=REFLECTION_MODEL, # Model for guideline distillation + retrieval_k=5, # Examples to retrieve per evaluation + embedding_model="databricks:/databricks-gte-large-en", + # Defaults to "openai/text-embedding-3-small" if not set -- see GOTCHAS.md +) + +# Load the registered base judge and run alignment +base_judge = get_scorer(name="domain_quality_base") +aligned_judge = base_judge.align( + traces=traces_for_alignment, + optimizer=optimizer, +) + +# Inspect distilled semantic guidelines — these encode expert preferences +print("Distilled Guidelines from SME feedback:") +for i, guideline in enumerate(aligned_judge._semantic_memory, 1): + print(f" {i}. {guideline.guideline_text}") + if guideline.source_trace_ids: + print(f" Derived from {len(guideline.source_trace_ids)} trace(s)") +``` + +--- + +## Pattern 5: Register the Aligned Judge + +Persist the aligned judge to the experiment for later retrieval in evaluation or optimization runs. + +```python +from mlflow.genai.scorers import ScorerSamplingConfig + +# Option A: Update the existing judge record in-place (recommended for iterative alignment) +aligned_judge_registered = aligned_judge.update( + experiment_id=EXPERIMENT_ID, + sampling_config=ScorerSamplingConfig(sample_rate=0.0), +) +print(f"Updated judge: {aligned_judge_registered.name}") + +# Option B: Register as a new named version (preserves the original for comparison) +from mlflow.genai.judges import make_judge + +aligned_judge_v2 = make_judge( + name="domain_quality_aligned_v1", + instructions=aligned_judge.instructions, # Includes distilled guidelines + feedback_value_type=float, # Match the original judge's feedback type + model=JUDGE_MODEL, +) +aligned_judge_v2 = aligned_judge_v2.register(experiment_id=EXPERIMENT_ID) + +# Retrieve in a later session +# NOTE: Episodic memory is lazily loaded — inspect .instructions, not ._episodic_memory +from mlflow.genai.scorers import get_scorer + +retrieved_judge = get_scorer(name="domain_quality_base", experiment_id=EXPERIMENT_ID) +print(retrieved_judge.instructions[:500]) # Shows aligned instructions with guidelines +``` + +--- + +## Pattern 6: Re-evaluate with Aligned Judge + +Run a fresh evaluation with the aligned judge. This gives a more accurate quality picture and establishes a baseline for prompt optimization if you choose to do that next. + +**Important: The aligned judge score may be lower than the unaligned judge score. This is expected and correct.** It means the aligned judge is now evaluating with domain-expert standards rather than generic best practices. A lower score from a more accurate judge is a better signal than a higher score from a judge that doesn't understand your domain. The optimization phase (`optimize_prompts()`) will improve the agent against this more accurate standard. + +```python +from mlflow.genai import evaluate +from mlflow.genai.scorers import get_scorer +from mlflow.genai.datasets import get_dataset + +aligned_judge = get_scorer(name="domain_quality_base", experiment_id=EXPERIMENT_ID) + +eval_dataset = get_dataset(name=DATASET_NAME) +df = eval_dataset.to_df() + +eval_records = [ + { + "inputs": { + "input": [{"role": "user", "content": extract_user_message(row)}] + } + } + for row in df["inputs"] +] + +with mlflow.start_run(run_name="aligned_judge_baseline"): + baseline_results = evaluate( + data=eval_records, + predict_fn=lambda input: AGENT.predict({"input": input}), + scorers=[aligned_judge], + ) + +print(f"Aligned judge baseline metrics: {baseline_results.metrics}") +# NOTE: If scores are lower than the unaligned judge, that is expected. +# The aligned judge is more accurate, not less generous. +``` + +--- + +## Using Aligned Judges Beyond Evaluation + +Aligned judges are not just for one-time evaluation. They can be used for: + +**Production monitoring:** +```python +from mlflow.genai.scorers import ScorerSamplingConfig + +aligned_judge = get_scorer(name="domain_quality_base", experiment_id=EXPERIMENT_ID) +monitoring_judge = aligned_judge.start( + sampling_config=ScorerSamplingConfig(sample_rate=0.1) # Score 10% of production traffic +) +``` + +**Prompt optimization input (see `patterns-prompt-optimization.md`):** +```python +# Pass the aligned judge as the scorer in optimize_prompts() +result = mlflow.genai.optimize_prompts( + predict_fn=predict_fn, + train_data=optimization_dataset, + prompt_uris=[prompt.uri], + optimizer=GepaPromptOptimizer(reflection_model=REFLECTION_MODEL), + scorers=[aligned_judge], # ← aligned judge drives GEPA's reflection +) +``` + +**Regression detection across agent versions:** +```python +with mlflow.start_run(run_name="agent_v2"): + v2_results = evaluate(data=eval_records, predict_fn=agent_v2, scorers=[aligned_judge]) + +# Metrics from aligned judge are more meaningful than unaligned LLM judge +``` diff --git a/.claude/skills/databricks-mlflow-evaluation/references/patterns-prompt-optimization.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-prompt-optimization.md new file mode 100644 index 00000000..01a79bd1 --- /dev/null +++ b/.claude/skills/databricks-mlflow-evaluation/references/patterns-prompt-optimization.md @@ -0,0 +1,163 @@ +# MLflow 3 Prompt Optimization with GEPA + +Patterns for automated prompt improvement using `optimize_prompts()` with the GEPA (Genetic-Pareto) optimizer. GEPA iteratively evolves a registered system prompt by evaluating candidates against a scorer, then promotes the best version. + +**Using an aligned judge as the scorer is recommended.** An aligned judge encodes domain-expert preferences, giving GEPA a more accurate optimization signal than a generic LLM judge. See `patterns-judge-alignment.md` for the full alignment workflow. + +For the full end-to-end loop (evaluate, label, align, optimize, promote), see `user-journeys.md` Journey 10. For details on the GEPA and MemAlign approaches, see the [Self-Optimizing Agent blog post](https://www.databricks.com/blog/self-optimizing-football-chatbot-guided-domain-experts-databricks). + +**Read `GOTCHAS.md` before implementing -- especially the GEPA sections.** + +--- + +## Pattern 1: Build Optimization Dataset (inputs + expectations required) + +GEPA requires both `inputs` AND `expectations` in every record. This is different from the eval dataset which only needs `inputs`. The `expectations` field is what GEPA uses during reflection to reason about why the current prompt is underperforming. + +```python +# optimization dataset must have both inputs AND expectations +optimization_dataset = [ + { + "inputs": { + "input": [{"role": "user", "content": "What are the tendencies on 3rd and short?"}] + }, + "expectations": { + "expected_response": ( + "The agent should identify key players and their 3rd-and-short involvement, " + "provide relevant statistics, and give tactical recommendations. " + "If data quality issues exist, they should be stated explicitly." + ) + } + }, + { + "inputs": { + "input": [{"role": "user", "content": "How does the offense perform against the blitz?"}] + }, + "expectations": { + "expected_response": ( + "The agent should analyze performance metrics vs. pressure, " + "compare success across different blitz packages, " + "and provide concrete defensive recommendations." + ) + } + }, + # Add 15-20 representative examples covering key use cases +] + +# Persist to MLflow dataset +from mlflow.genai.datasets import create_dataset + +optim_dataset = create_dataset(name=OPTIMIZATION_DATASET_NAME) +optim_dataset = optim_dataset.merge_records(optimization_dataset) +print(f"Created optimization dataset with {len(optimization_dataset)} records") +``` + +--- + +## Pattern 2: Run optimize_prompts() with GEPA + +Use a scorer (ideally an aligned judge from `patterns-judge-alignment.md`) to drive GEPA prompt optimization of the registered system prompt. + +```python +import mlflow +from mlflow.genai.optimize import GepaPromptOptimizer +from mlflow.genai.scorers import get_scorer + +mlflow.set_experiment(experiment_id=EXPERIMENT_ID) + +# Load prompt from registry (must be registered before optimization) +system_prompt = mlflow.genai.load_prompt(f"prompts:/{PROMPT_NAME}@production") +print(f"Loaded prompt: {system_prompt.uri}") + +# Load scorer -- an aligned judge is recommended for domain-accurate optimization +# See patterns-judge-alignment.md for how to create one +aligned_judge = get_scorer(name=ALIGNED_JUDGE_NAME, experiment_id=EXPERIMENT_ID) + +# Define predict_fn -- loads prompt from registry on each call so GEPA can swap it +def predict_fn(input): + prompt = mlflow.genai.load_prompt(system_prompt.uri) + system_content = prompt.format() + + user_message = input[0]["content"] + messages = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": user_message}, + ] + return AGENT.predict({"input": messages}) + +# Define aggregation to normalize judge feedback (Feedback.value) to 0-1 for GEPA +def objective_function(scores: dict) -> float: + feedback = scores.get(ALIGNED_JUDGE_NAME) + if feedback and hasattr(feedback, "feedback") and hasattr(feedback.feedback, "value"): + try: + return float(feedback.feedback.value) / 5.0 # Normalize 1-5 scale to 0-1 + except (ValueError, TypeError): + return 0.5 + return 0.5 + +# Run optimization +result = mlflow.genai.optimize_prompts( + predict_fn=predict_fn, + train_data=optimization_dataset, # Must have inputs + expectations + prompt_uris=[system_prompt.uri], + optimizer=GepaPromptOptimizer( + reflection_model=REFLECTION_MODEL, + max_metric_calls=75, # Reduce for faster runs; increase for quality + display_progress_bar=True, + ), + scorers=[aligned_judge], + aggregation=objective_function, +) + +optimized_prompt = result.optimized_prompts[0] +print(f"Initial score: {result.initial_eval_score}") +print(f"Final score: {result.final_eval_score}") +print(f"\nOptimized template (first 500 chars):\n{optimized_prompt.template[:500]}...") +``` + +--- + +## Pattern 3: Register Optimized Prompt and Conditionally Promote + +Only promote to the "production" alias if the optimized prompt outperforms the baseline. + +```python +# Register new prompt version with optimization metadata +new_prompt_version = mlflow.genai.register_prompt( + name=PROMPT_NAME, + template=optimized_prompt.template, + commit_message=f"GEPA optimization using {ALIGNED_JUDGE_NAME}", + tags={ + "initial_score": str(result.initial_eval_score), + "final_score": str(result.final_eval_score), + "optimization": "GEPA", + "judge": ALIGNED_JUDGE_NAME, + }, +) +print(f"Registered prompt version: {new_prompt_version.version}") + +# Conditional promotion -- only update production alias if score improved +def promote_if_improved(prompt_name, result, new_prompt_version): + if result.final_eval_score > result.initial_eval_score: + mlflow.genai.set_prompt_alias( + name=prompt_name, + alias="production", + version=new_prompt_version.version, + ) + print(f"Promoted version {new_prompt_version.version} to production " + f"({result.initial_eval_score:.3f} -> {result.final_eval_score:.3f})") + else: + print(f"No improvement ({result.initial_eval_score:.3f} -> " + f"{result.final_eval_score:.3f}). Production alias unchanged.") + +promote_if_improved(PROMPT_NAME, result, new_prompt_version) +``` + +--- + +## Tips for Prompt Optimization + +- The optimization dataset should cover the diversity of queries your agent will handle. Include edge cases, ambiguous requests, and scenarios where tool selection matters. +- Expected responses should describe what the agent should do (which tools to call, what information to include) rather than exact output text. +- Start with `max_metric_calls` set to between 50 and 100. Higher values explore more candidates but increase cost and runtime. +- The GEPA optimizer learns from failure modes. If the aligned judge penalizes missing benchmarks or small-sample caveats, GEPA will inject those requirements into the optimized prompt. diff --git a/.claude/skills/mlflow-evaluation/references/patterns-scorers.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-scorers.md similarity index 100% rename from .claude/skills/mlflow-evaluation/references/patterns-scorers.md rename to .claude/skills/databricks-mlflow-evaluation/references/patterns-scorers.md diff --git a/.claude/skills/mlflow-evaluation/references/patterns-trace-analysis.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-trace-analysis.md similarity index 100% rename from .claude/skills/mlflow-evaluation/references/patterns-trace-analysis.md rename to .claude/skills/databricks-mlflow-evaluation/references/patterns-trace-analysis.md diff --git a/.claude/skills/databricks-mlflow-evaluation/references/patterns-trace-ingestion.md b/.claude/skills/databricks-mlflow-evaluation/references/patterns-trace-ingestion.md new file mode 100644 index 00000000..7196ab14 --- /dev/null +++ b/.claude/skills/databricks-mlflow-evaluation/references/patterns-trace-ingestion.md @@ -0,0 +1,680 @@ +# MLflow Trace Ingestion in Unity Catalog + +Working code patterns for setting up trace storage in Unity Catalog, logging traces from applications, and enabling production monitoring. + +**Version**: MLflow 3.9.0+ (`mlflow[databricks]>=3.9.0`) +**Preview**: Requires "OpenTelemetry on Databricks" preview enabled +**Regions**: Currently available in `us-east-1` and `us-west-2` only + +--- + +## Table of Contents + +| # | Pattern | Description | +|---|---------|-------------| +| 1 | [Initial Setup](#pattern-1-initial-setup---link-uc-schema-to-experiment) | Link UC schema to experiment, create tables | +| 2 | [Access Control](#pattern-2-access-control---grant-permissions) | Grant required permissions on UC tables | +| 3 | [Set Trace Destination (Python API)](#pattern-3-set-trace-destination-via-python-api) | Configure where traces are sent | +| 4 | [Set Trace Destination (Env Var)](#pattern-4-set-trace-destination-via-environment-variable) | Configure destination via env var | +| 5 | [Log Traces with @mlflow.trace](#pattern-5-log-traces-with-mlflow-decorator) | Instrument functions with decorator | +| 6 | [Log Traces with start_span](#pattern-6-log-traces-with-context-manager) | Fine-grained span control | +| 7 | [Auto-Instrumentation](#pattern-7-automatic-tracing-with-autolog) | Framework auto-tracing (OpenAI, LangChain, etc.) | +| 8 | [Combined Instrumentation](#pattern-8-combined-auto-and-manual-tracing) | Mix auto + manual tracing | +| 9 | [Traces from Databricks Apps](#pattern-9-log-traces-from-databricks-apps) | Configure app service principal | +| 10 | [Traces from Model Serving](#pattern-10-log-traces-from-model-serving-endpoints) | Configure serving endpoints | +| 11 | [Traces from OTEL Clients](#pattern-11-log-traces-from-third-party-otel-clients) | Use OpenTelemetry OTLP exporter | +| 12 | [Enable Production Monitoring](#pattern-12-enable-production-monitoring) | Register and start scorers | +| 13 | [Manage Monitoring Scorers](#pattern-13-manage-monitoring-scorers) | List, update, stop, delete scorers | +| 14 | [Query UC Trace Tables](#pattern-14-query-traces-from-unity-catalog-tables) | SQL queries on ingested traces | +| 15 | [End-to-End Setup](#pattern-15-end-to-end-setup-script) | Complete setup from scratch | + +--- + +## Pattern 1: Initial Setup - Link UC Schema to Experiment + +Create an MLflow experiment and link it to a Unity Catalog schema. This automatically creates three tables for storing trace data. + +```python +import os +import mlflow +from mlflow.entities import UCSchemaLocation +from mlflow.tracing.enablement import set_experiment_trace_location + +# Step 1: Configure tracking +mlflow.set_tracking_uri("databricks") +os.environ["MLFLOW_TRACING_SQL_WAREHOUSE_ID"] = "" + +# Step 2: Define names +experiment_name = "/Shared/my-agent-traces" +catalog_name = "my_catalog" +schema_name = "my_schema" + +# Step 3: Create or retrieve experiment +if experiment := mlflow.get_experiment_by_name(experiment_name): + experiment_id = experiment.experiment_id +else: + experiment_id = mlflow.create_experiment(name=experiment_name) + +# Step 4: Link UC schema to experiment +result = set_experiment_trace_location( + location=UCSchemaLocation( + catalog_name=catalog_name, + schema_name=schema_name + ), + experiment_id=experiment_id, +) +``` + +**Tables created automatically:** +- `{catalog}.{schema}.mlflow_experiment_trace_otel_logs` +- `{catalog}.{schema}.mlflow_experiment_trace_otel_metrics` +- `{catalog}.{schema}.mlflow_experiment_trace_otel_spans` + +**CRITICAL**: Linking a UC schema hides pre-existing experiment traces stored in MLflow. Unlinking restores access to those traces. + +--- + +## Pattern 2: Access Control - Grant Permissions + +Users and service principals need explicit permissions on the UC trace tables. `ALL_PRIVILEGES` is **not sufficient**. + +```sql +-- Required: USE_CATALOG on the catalog +GRANT USE_CATALOG ON CATALOG my_catalog TO `user@company.com`; + +-- Required: USE_SCHEMA on the schema +GRANT USE_SCHEMA ON SCHEMA my_catalog.my_schema TO `user@company.com`; + +-- Required: MODIFY and SELECT on each trace table +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_logs + TO `user@company.com`; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_spans + TO `user@company.com`; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_metrics + TO `user@company.com`; +``` + +**For service principals (Databricks Apps, Model Serving):** +```sql +-- Replace with the service principal's application ID +GRANT USE_CATALOG ON CATALOG my_catalog TO ``; +GRANT USE_SCHEMA ON SCHEMA my_catalog.my_schema TO ``; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_logs + TO ``; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_spans + TO ``; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_metrics + TO ``; +``` + +--- + +## Pattern 3: Set Trace Destination via Python API + +Configure where traces are sent using the Python API. Use this after the initial setup (Pattern 1) in your application code. + +```python +import mlflow +from mlflow.entities import UCSchemaLocation + +# Set trace destination to Unity Catalog +mlflow.tracing.set_destination( + destination=UCSchemaLocation( + catalog_name="my_catalog", + schema_name="my_schema", + ) +) + +# Now all traces from @mlflow.trace or autolog will go to UC +@mlflow.trace +def my_agent(query: str) -> str: + # Traces are automatically sent to UC tables + return process(query) +``` + +--- + +## Pattern 4: Set Trace Destination via Environment Variable + +Alternative to Pattern 3 — configure destination via environment variable. Useful for deployment configurations. + +```python +import os + +# Set destination as "{catalog}.{schema}" +os.environ["MLFLOW_TRACING_DESTINATION"] = "my_catalog.my_schema" +``` + +Or in shell: +```bash +export MLFLOW_TRACING_DESTINATION="my_catalog.my_schema" +``` + +--- + +## Pattern 5: Log Traces with MLflow Decorator + +Use `@mlflow.trace` to instrument functions. Automatically captures inputs, outputs, latency, and exceptions. + +```python +import mlflow +from mlflow.entities import SpanType + +# Basic function tracing +@mlflow.trace +def my_agent(query: str) -> str: + context = retrieve_context(query) + return generate_response(query, context) + +# With span type (enables enhanced UI and evaluation) +@mlflow.trace(span_type=SpanType.RETRIEVER) +def retrieve_context(query: str) -> list[dict]: + """Mark retrieval functions with RETRIEVER span type.""" + return vector_store.search(query, top_k=5) + +@mlflow.trace(span_type=SpanType.CHAIN) +def generate_response(query: str, context: list[dict]) -> str: + """Mark orchestration with CHAIN span type.""" + return llm.invoke(query, context=context) + +# With custom name and attributes +@mlflow.trace(name="safety_check", span_type=SpanType.TOOL) +def check_safety(text: str) -> bool: + return safety_classifier.predict(text) +``` + +**Available SpanType values:** +- `SpanType.CHAIN` — Orchestration / pipeline steps +- `SpanType.CHAT_MODEL` — LLM chat completions +- `SpanType.LLM` — LLM calls (non-chat) +- `SpanType.RETRIEVER` — Document/data retrieval (special output schema) +- `SpanType.TOOL` — Tool/function execution +- `SpanType.AGENT` — Agent execution +- `SpanType.EMBEDDING` — Embedding generation + +--- + +## Pattern 6: Log Traces with Context Manager + +Use `mlflow.start_span()` for fine-grained control over spans. Manually set inputs, outputs, and attributes. + +```python +import mlflow + +def process_query(query: str) -> str: + # Create a span with manual control + with mlflow.start_span(name="process_query") as span: + span.set_inputs({"query": query}) + + # Nested span for retrieval + with mlflow.start_span(name="retrieve", span_type="RETRIEVER") as retriever_span: + retriever_span.set_inputs({"query": query}) + docs = vector_store.search(query) + retriever_span.set_outputs(docs) + + # Nested span for generation + with mlflow.start_span(name="generate", span_type="CHAIN") as gen_span: + gen_span.set_inputs({"query": query, "doc_count": len(docs)}) + response = llm.generate(query, docs) + gen_span.set_outputs({"response": response}) + + # Set attributes for analysis + span.set_attribute("doc_count", len(docs)) + span.set_attribute("model", "gpt-4o") + span.set_outputs({"response": response}) + + return response +``` + +--- + +## Pattern 7: Automatic Tracing with Autolog + +Enable automatic tracing for supported frameworks. MLflow captures LLM calls, tool executions, and chain operations without code changes. + +```python +import mlflow + +# Enable auto-tracing for specific frameworks +mlflow.openai.autolog() # OpenAI SDK calls +mlflow.langchain.autolog() # LangChain chains and agents +# Also available: mlflow.anthropic.autolog(), mlflow.litellm.autolog(), etc. + +# Set tracking and destination +mlflow.set_tracking_uri("databricks") +mlflow.set_experiment("/Shared/my-agent-traces") + +# Traces are captured automatically +from openai import OpenAI +client = OpenAI() + +response = client.chat.completions.create( + model="gpt-4o", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is MLflow?"} + ] +) +# ^ This call is automatically traced +``` + +**20+ supported frameworks** including: +- OpenAI, Anthropic, Google GenAI +- LangChain, LlamaIndex, DSPy +- LiteLLM, Ollama, Bedrock +- CrewAI, AutoGen, Haystack + +--- + +## Pattern 8: Combined Auto and Manual Tracing + +Combine automatic framework tracing with manual decorators for complete coverage. + +```python +import mlflow +from mlflow.entities import SpanType +from openai import OpenAI + +# Enable automatic OpenAI tracing +mlflow.openai.autolog() + +client = OpenAI() + +@mlflow.trace(span_type=SpanType.CHAIN) +def my_rag_pipeline(query: str) -> str: + """Manual decorator wraps the whole pipeline. + Auto-tracing captures individual OpenAI calls inside.""" + + # This retrieval is manually traced + docs = retrieve_documents(query) + + # This LLM call is auto-traced by mlflow.openai.autolog() + response = client.chat.completions.create( + model="gpt-4o", + messages=[ + {"role": "system", "content": f"Answer using context: {docs}"}, + {"role": "user", "content": query} + ] + ) + return response.choices[0].message.content + +@mlflow.trace(span_type=SpanType.RETRIEVER) +def retrieve_documents(query: str) -> list[dict]: + """Manually traced retrieval function.""" + return vector_store.search(query, top_k=5) +``` + +--- + +## Pattern 9: Log Traces from Databricks Apps + +Configure a Databricks App to send traces to Unity Catalog. + +**Prerequisites:** +- App uses `mlflow[databricks]>=3.5.0` +- App's service principal has MODIFY and SELECT on the trace tables (see Pattern 2) + +**In your app code:** +```python +import os +import mlflow +from mlflow.entities import UCSchemaLocation + +# Option A: Python API +mlflow.tracing.set_destination( + destination=UCSchemaLocation( + catalog_name="my_catalog", + schema_name="my_schema", + ) +) + +# Option B: Environment variable (set in app config) +os.environ["MLFLOW_TRACING_DESTINATION"] = "my_catalog.my_schema" + +# Your app code — traces are sent to UC +@mlflow.trace +def handle_request(query: str) -> str: + return my_agent.invoke(query) +``` + +**Deployment steps:** +1. Locate the app's service principal under the **Authorization** tab +2. Grant MODIFY and SELECT on the three `mlflow_experiment_trace_*` tables +3. Configure the trace destination in your app code +4. Deploy the app + +--- + +## Pattern 10: Log Traces from Model Serving Endpoints + +Configure a model serving endpoint to send traces to Unity Catalog. + +**Step 1: Grant permissions to user/service principal** +```sql +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_logs + TO `serving-principal-id`; +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_spans + TO `serving-principal-id`; +``` + +**Step 2: Generate a Personal Access Token (PAT)** + +Create a PAT for the identity that has the permissions above. + +**Step 3: Add environment variables to the endpoint** + +Add these to the serving endpoint configuration: +``` +DATABRICKS_TOKEN= +MLFLOW_TRACING_DESTINATION=my_catalog.my_schema +``` + +**Step 4: In your served model code, configure the destination** +```python +import os +import mlflow +from mlflow.entities import UCSchemaLocation + +mlflow.tracing.set_destination( + destination=UCSchemaLocation( + catalog_name="my_catalog", + schema_name="my_schema", + ) +) + +# Your model's predict function — traces go to UC +@mlflow.trace +def predict(model_input): + return my_model.invoke(model_input) +``` + +--- + +## Pattern 11: Log Traces from Third-Party OTEL Clients + +Send traces from any OpenTelemetry-compatible client to Unity Catalog via the OTLP HTTP endpoint. + +```python +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +# Configure OTLP exporter pointing to Databricks +otlp_trace_exporter = OTLPSpanExporter( + endpoint="https:///api/2.0/otel/v1/traces", + headers={ + "content-type": "application/x-protobuf", + "X-Databricks-UC-Table-Name": "my_catalog.my_schema.mlflow_experiment_trace_otel_spans", + "Authorization": "Bearer ", + }, +) + +# Set up the tracer provider +provider = TracerProvider() +provider.add_span_processor(BatchSpanProcessor(otlp_trace_exporter)) + +# Use standard OpenTelemetry APIs to create spans +tracer = provider.get_tracer("my-application") +with tracer.start_as_current_span("my-operation") as span: + span.set_attribute("query", "What is MLflow?") + result = process_query("What is MLflow?") + span.set_attribute("result_length", len(result)) +``` + +**Notes:** +- Traces ingested via OTEL appear in linked experiments if they contain a root span +- Use the `X-Databricks-UC-Table-Name` header to specify the target spans table +- Standard OTEL instrumentation libraries work with this endpoint + +--- + +## Pattern 12: Enable Production Monitoring + +Register scorers to continuously evaluate traces in production. Scorers run asynchronously on sampled traces. + +```python +import mlflow +from mlflow.genai.scorers import Safety, Guidelines, ScorerSamplingConfig +from mlflow.tracing import set_databricks_monitoring_sql_warehouse_id + +# Step 1: Configure the SQL warehouse for monitoring +set_databricks_monitoring_sql_warehouse_id( + warehouse_id="", + experiment_id="" # Optional — uses active experiment if omitted +) + +# Step 2: Set the active experiment +mlflow.set_experiment("/Shared/my-agent-traces") + +# Step 3: Register and start scorers + +# Safety scorer — evaluate 100% of traces +safety = Safety().register(name="production_safety") +safety = safety.start( + sampling_config=ScorerSamplingConfig(sample_rate=1.0) +) + +# Custom guidelines — evaluate 50% of traces +tone_check = Guidelines( + name="professional_tone", + guidelines="The response must be professional and helpful" +).register(name="production_tone") +tone_check = tone_check.start( + sampling_config=ScorerSamplingConfig(sample_rate=0.5) +) +``` + +**CRITICAL**: You must both `.register()` AND `.start()` — registering alone does not activate monitoring. + +**SQL Warehouse requirements:** +- User must have `CAN USE` on the SQL warehouse +- User must have `CAN EDIT` on the experiment +- Monitoring job permissions are auto-granted on first scorer registration + +--- + +## Pattern 13: Manage Monitoring Scorers + +List, update, stop, and delete production monitoring scorers. + +```python +from mlflow.genai.scorers import list_scorers, get_scorer, delete_scorer, ScorerSamplingConfig + +# List all registered scorers for the active experiment +scorers = list_scorers() +for s in scorers: + print(f" {s.name}: sample_rate={s.sampling_config.sample_rate if s.sampling_config else 'N/A'}") + +# Get a specific scorer +safety_scorer = get_scorer(name="production_safety") + +# Update sample rate (e.g., increase from 50% to 80%) +safety_scorer = safety_scorer.update( + sampling_config=ScorerSamplingConfig(sample_rate=0.8) +) + +# Stop monitoring (keeps registration for later re-start) +safety_scorer = safety_scorer.stop() + +# Re-start monitoring +safety_scorer = safety_scorer.start( + sampling_config=ScorerSamplingConfig(sample_rate=0.5) +) + +# Delete entirely (removes registration) +delete_scorer(name="production_safety") +``` + +--- + +## Pattern 14: Query Traces from Unity Catalog Tables + +Query ingested traces directly using SQL for custom analysis and dashboards. + +```sql +-- Count traces per day +SELECT + DATE(timestamp) as trace_date, + COUNT(DISTINCT trace_id) as trace_count +FROM my_catalog.my_schema.mlflow_experiment_trace_otel_spans +WHERE parent_span_id IS NULL -- root spans only +GROUP BY DATE(timestamp) +ORDER BY trace_date DESC; + +-- Find slow traces (root span duration > 10s) +SELECT + trace_id, + name as root_span_name, + (end_time_unix_nano - start_time_unix_nano) / 1e9 as duration_seconds +FROM my_catalog.my_schema.mlflow_experiment_trace_otel_spans +WHERE parent_span_id IS NULL + AND (end_time_unix_nano - start_time_unix_nano) / 1e9 > 10 +ORDER BY duration_seconds DESC +LIMIT 20; + +-- Error rate by span name +SELECT + name, + COUNT(*) as total, + SUM(CASE WHEN status_code = 'ERROR' THEN 1 ELSE 0 END) as errors, + ROUND(SUM(CASE WHEN status_code = 'ERROR' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 2) as error_pct +FROM my_catalog.my_schema.mlflow_experiment_trace_otel_spans +GROUP BY name +HAVING COUNT(*) > 10 +ORDER BY error_pct DESC; +``` + +**From Python (via Spark):** +```python +from databricks.connect import DatabricksSession + +spark = DatabricksSession.builder.remote(serverless=True).getOrCreate() + +# Query trace spans +spans_df = spark.sql(""" + SELECT trace_id, name, span_kind, + (end_time_unix_nano - start_time_unix_nano) / 1e6 as duration_ms + FROM my_catalog.my_schema.mlflow_experiment_trace_otel_spans + WHERE name LIKE '%retriever%' + ORDER BY duration_ms DESC + LIMIT 100 +""") +spans_df.show() +``` + +--- + +## Pattern 15: End-to-End Setup Script + +Complete setup script for a new project — from creating the UC schema link to logging the first trace and enabling monitoring. + +```python +import os +import mlflow +from mlflow.entities import UCSchemaLocation +from mlflow.tracing.enablement import set_experiment_trace_location +from mlflow.tracing import set_databricks_monitoring_sql_warehouse_id +from mlflow.genai.scorers import Safety, Guidelines, ScorerSamplingConfig + +# ============================================================ +# Configuration — UPDATE THESE VALUES +# ============================================================ +EXPERIMENT_NAME = "/Shared/my-agent-traces" +CATALOG_NAME = "my_catalog" +SCHEMA_NAME = "my_schema" +SQL_WAREHOUSE_ID = "abc123def456" # Your SQL warehouse ID + +# ============================================================ +# Step 1: Initial Setup +# ============================================================ +mlflow.set_tracking_uri("databricks") +os.environ["MLFLOW_TRACING_SQL_WAREHOUSE_ID"] = SQL_WAREHOUSE_ID + +# Create or retrieve experiment +if experiment := mlflow.get_experiment_by_name(EXPERIMENT_NAME): + experiment_id = experiment.experiment_id +else: + experiment_id = mlflow.create_experiment(name=EXPERIMENT_NAME) + +# Link UC schema (creates trace tables automatically) +set_experiment_trace_location( + location=UCSchemaLocation( + catalog_name=CATALOG_NAME, + schema_name=SCHEMA_NAME + ), + experiment_id=experiment_id, +) +print(f"Linked experiment '{EXPERIMENT_NAME}' to {CATALOG_NAME}.{SCHEMA_NAME}") + +# ============================================================ +# Step 2: Set Trace Destination +# ============================================================ +mlflow.set_experiment(EXPERIMENT_NAME) +mlflow.tracing.set_destination( + destination=UCSchemaLocation( + catalog_name=CATALOG_NAME, + schema_name=SCHEMA_NAME, + ) +) + +# ============================================================ +# Step 3: Enable Production Monitoring +# ============================================================ +set_databricks_monitoring_sql_warehouse_id( + warehouse_id=SQL_WAREHOUSE_ID, + experiment_id=experiment_id, +) + +# Register and start safety monitoring (100% of traces) +safety = Safety().register(name="safety_monitor") +safety = safety.start( + sampling_config=ScorerSamplingConfig(sample_rate=1.0) +) +print("Safety monitoring enabled (100% sample rate)") + +# Register and start custom guidelines (50% of traces) +tone = Guidelines( + name="professional_tone", + guidelines="The response must be professional, helpful, and concise" +).register(name="tone_monitor") +tone = tone.start( + sampling_config=ScorerSamplingConfig(sample_rate=0.5) +) +print("Tone monitoring enabled (50% sample rate)") + +# ============================================================ +# Step 4: Verify with a Test Trace +# ============================================================ +@mlflow.trace +def test_agent(query: str) -> str: + return f"Test response to: {query}" + +result = test_agent("Hello, is tracing working?") +print(f"Test trace logged. Check the Experiments UI at: {EXPERIMENT_NAME}") +``` + +--- + +## Limitations & Quotas + +| Limit | Value | +|-------|-------| +| Trace ingestion rate | 100 traces/second per workspace | +| Table ingestion throughput | 100 MB/second per table | +| Query throughput | 200 queries/second | +| UI performance | Degrades with >2TB of data | +| Trace deletion | Individual deletion not supported (use SQL) | +| MLflow MCP server | Does not support UC-stored traces | +| Region availability | `us-east-1` and `us-west-2` only (Beta) | + +--- + +## Viewing Traces in the UI + +1. Navigate to the **Experiments** page in your Databricks workspace +2. Select your experiment +3. Click the **Traces** tab +4. Select a **SQL warehouse** from the dropdown to query UC-stored traces +5. Browse traces, inspect spans, view inputs/outputs + +**Note:** You must select a SQL warehouse to view UC-stored traces — they are not loaded automatically. diff --git a/.claude/skills/databricks-mlflow-evaluation/references/user-journeys.md b/.claude/skills/databricks-mlflow-evaluation/references/user-journeys.md new file mode 100644 index 00000000..6ff09b28 --- /dev/null +++ b/.claude/skills/databricks-mlflow-evaluation/references/user-journeys.md @@ -0,0 +1,627 @@ +# User Journey Guides + +Step-by-step workflows for common evaluation scenarios. + +--- + +## Journey 0: Strategy Alignment (ALWAYS START HERE) + +**Starting Point**: You need to evaluate an agent +**Goal**: Align on what to evaluate before writing any code + +**PRIORITY:** Before writing evaluation code, complete strategy alignment. This ensures evaluations measure what matters and provide actionable insights. + +### Step 1: Understand the Agent + +Before evaluating, gather context about what you're evaluating: + +**Questions to ask (or investigate in the codebase):** +1. **What does this agent do?** (data analysis, RAG, multi-turn chat, task automation) +2. **What tools does it use?** (UC functions, vector search, external APIs) +3. **What is the input/output format?** (messages format, structured output) +4. **What is the current state?** (prototype, production, needs improvement) + +**Actions to take:** +- Read the agent's main code file (e.g., `agent.py`) +- Review the config file for system prompts and tool definitions +- Check existing tests or evaluation scripts +- Look at CLAUDE.md or README for project context + +### Step 2: Align on What to Evaluate + +**Evaluation dimensions to consider:** + +| Dimension | When to Use | Example Scorer | +|-----------|-------------|----------------| +| **Safety** | Always (table stakes) | `Safety()` | +| **Correctness** | When ground truth exists | `Correctness()` | +| **Relevance** | When responses should address queries | `RelevanceToQuery()` | +| **Groundedness** | RAG systems with retrieved context | `RetrievalGroundedness()` | +| **Domain Guidelines** | Domain-specific requirements | `Guidelines(name="...", guidelines="...")` | +| **Format/Structure** | Structured output requirements | Custom scorer | +| **Tool Usage** | Agents with tool calls | Custom scorer checking tool selection | + +**Questions to ask the user:** +1. What are the **must-have** quality criteria? (safety, accuracy, relevance) +2. What are the **nice-to-have** criteria? (conciseness, tone, format) +3. Are there **specific failure modes** you've seen or worry about? +4. Do you have **ground truth** or expected answers for test cases? + +### Step 3: Define User Scenarios (Evaluation Dataset) + +**Types of test cases to include:** + +| Category | Purpose | Example | +|----------|---------|---------| +| **Happy Path** | Core functionality works | Typical user questions | +| **Edge Cases** | Boundary conditions | Empty inputs, very long queries | +| **Adversarial** | Robustness testing | Prompt injection, off-topic | +| **Multi-turn** | Conversation handling | Follow-up questions, context recall | +| **Domain-specific** | Business logic | Industry terminology, specific formats | + +**Questions to ask the user:** +1. What are the **most common** questions users ask? +2. What are **challenging** questions the agent should handle? +3. Are there questions it should **refuse** to answer? +4. Do you have **existing test cases** or production traces to start from? + +### Step 4: Establish Success Criteria + +**Define quality gates before running evaluation:** + +```python +QUALITY_GATES = { + "safety": 1.0, # 100% - non-negotiable + "correctness": 0.9, # 90% - high bar for accuracy + "relevance": 0.85, # 85% - good relevance + "concise": 0.8, # 80% - nice to have +} +``` + +**Questions to ask the user:** +1. What pass rates are **acceptable** for each dimension? +2. Which metrics are **blocking** vs **informational**? +3. How will evaluation results **inform decisions**? (ship/no-ship, iterate, investigate) + +### Strategy Alignment Checklist + +Before implementing evaluation, confirm: +- [ ] Agent purpose and architecture understood +- [ ] Evaluation dimensions agreed upon +- [ ] Test case categories identified +- [ ] Success criteria defined +- [ ] Data source identified (new, traces, existing dataset) + +--- + +## Journey 3: "Something Broke" - Regression Detection + +**Starting Point**: You made changes to your agent and suspect something regressed +**Goal**: Identify what broke and verify the fix + +### Steps + +1. **Establish baseline metrics** + ```bash + # Run evaluation on the previous version (or use saved baseline) + cd agents/tool_calling_dspy + python run_quick_eval.py + ``` + Record key metrics: `classifier_accuracy`, `tool_selection_accuracy`, `follows_instructions` + +2. **Run evaluation on current version** + ```bash + python run_quick_eval.py + ``` + +3. **Compare metrics** + ```python + from evaluation.optimization_history import OptimizationHistory + + history = OptimizationHistory() + print(history.compare_iterations(-2, -1)) # Compare last two + ``` + +4. **Identify regression source** + - If `classifier_accuracy` dropped → Check ClassifierSignature changes + - If `tool_selection_accuracy` dropped → Check tool descriptions, required_tools field + - If `follows_instructions` dropped → Check ExecutorSignature output format + +5. **Analyze failing traces** + ``` + /eval:analyze-traces [experiment-id] + ``` + Look for: + - Error patterns in specific test categories + - Tool call failures + - Unexpected outputs + +6. **Fix and re-evaluate** + - Revert problematic changes or apply targeted fix + - Re-run evaluation + - Verify metrics restored + +### Commands Used +- `python run_quick_eval.py` - Run evaluation +- `/eval:analyze-traces` - Deep trace analysis +- `OptimizationHistory.compare_iterations()` - Metric comparison + +### Success Indicators +- Metrics return to baseline or improve +- No new failing test cases +- Trace analysis shows expected behavior + +--- + +## Journey 7: "My Multi-Agent is Slow" - Performance Optimization + +**Starting Point**: Your agent responses are too slow +**Goal**: Identify bottlenecks and reduce latency + +### Steps + +1. **Run evaluation with latency scoring** + ```bash + cd agents/tool_calling_dspy + python run_quick_eval.py + ``` + Note the latency metrics: + - `classifier_latency_ms` + - `rewriter_latency_ms` + - `executor_latency_ms` + - `total_latency_ms` + +2. **Identify the bottleneck stage** + | Latency | Typical Range | If High, Check | + |---------|---------------|----------------| + | classifier_latency | <5s | ClassifierSignature verbosity | + | rewriter_latency | <10s | QueryRewriterSignature complexity | + | executor_latency | <30s | Tool call count, response generation | + +3. **Analyze traces for slow stages** + ``` + /eval:analyze-traces [experiment-id] + ``` + Focus on: + - Span durations by stage + - Number of LLM calls per stage + - Tool execution times + +4. **Run signature analysis** + ```bash + python -m evaluation.analyze_signatures + ``` + Look for: + - High total description chars (>2000) + - Verbose OutputField descriptions + - Missing examples (causes more retries) + +5. **Apply optimizations** + + **For high classifier latency:** + - Simplify ClassifierSignature docstring + - Add concrete examples to reduce ambiguity + + **For high executor latency:** + - Simplify ExecutorSignature.answer format + - Reduce output format requirements + - Consider caching repeated tool calls + + **For high total latency:** + - Review if all stages are necessary + - Consider parallel execution where possible + +6. **Re-evaluate and compare** + ```bash + python run_quick_eval.py + ``` + Use `OptimizationHistory.compare_iterations()` to verify improvement + +### Commands Used +- `python run_quick_eval.py` - Run evaluation with latency scoring +- `/eval:analyze-traces` - Trace analysis with timing breakdown +- `python -m evaluation.analyze_signatures` - Signature verbosity analysis + +### Success Indicators +- Target latencies: classifier <5s, executor <30s, total <60s +- No regression in accuracy metrics +- Consistent improvement across test categories + +--- + +## Journey 8: "Improve My Prompts" - Systematic Prompt Optimization + +**Starting Point**: Your agent works but could be more accurate +**Goal**: Systematically improve prompt quality through evaluation + +### Steps + +1. **Establish baseline** + ```bash + cd agents/tool_calling_dspy + python run_quick_eval.py + ``` + Record all metrics in `optimization_history.json` + +2. **Run signature analysis** + ```bash + python -m evaluation.analyze_signatures + ``` + Review the report for: + - Metric correlations (which signatures affect which metrics) + - Specific issues flagged per signature + +3. **Prioritize fixes by metric impact** + + | Metric | Primary Signature | Common Issues | + |--------|-------------------|---------------| + | follows_instructions | ExecutorSignature | Verbose answer format, unclear structure | + | tool_selection_accuracy | ClassifierSignature | No examples, ambiguous tool descriptions | + | classifier_accuracy | ClassifierSignature | Verbose docstring, unclear query_type mapping | + +4. **Apply ONE fix at a time** + - Make a single, targeted change + - Document the change in your commit message + - Track in optimization_history.json + +5. **Re-evaluate immediately** + ```bash + python run_quick_eval.py + ``` + - If improved → Keep change, move to next fix + - If regressed → Revert and try different approach + - If unchanged → Consider if fix was necessary + +6. **Iterate until targets met** + + | Metric | Target | + |--------|--------| + | classifier_accuracy | 95%+ | + | tool_selection_accuracy | 90%+ | + | follows_instructions | 80%+ | + +7. **Document successful optimizations** + ```python + from evaluation.optimization_history import OptimizationHistory + + history = OptimizationHistory() + print(history.summary()) + ``` + +### Commands Used +- `python run_quick_eval.py` - Run evaluation +- `python -m evaluation.analyze_signatures` - Identify prompt issues +- `/optimize:context --quick` - Full optimization loop (when endpoint available) + +### Success Indicators +- All target metrics met +- No regressions from baseline +- Clear documentation of what changed and why +- Optimization history shows positive trend + +--- + +## Journey 9: "Store Traces in Unity Catalog" - Trace Ingestion & Production Monitoring + +**Starting Point**: You want to persist traces in Unity Catalog for long-term analysis, compliance, or production monitoring +**Goal**: Set up trace ingestion, instrument your app, and enable continuous monitoring + +### Prerequisites + +- Unity Catalog-enabled workspace +- "OpenTelemetry on Databricks" preview enabled +- SQL warehouse with `CAN USE` permissions +- MLflow 3.9.0+ (`pip install mlflow[databricks]>=3.9.0`) +- Workspace in `us-east-1` or `us-west-2` (Beta limitation) + +### Steps + +1. **Link UC schema to experiment** + ```python + import os + import mlflow + from mlflow.entities import UCSchemaLocation + from mlflow.tracing.enablement import set_experiment_trace_location + + mlflow.set_tracking_uri("databricks") + os.environ["MLFLOW_TRACING_SQL_WAREHOUSE_ID"] = "" + + experiment_id = mlflow.create_experiment(name="/Shared/my-traces") + set_experiment_trace_location( + location=UCSchemaLocation(catalog_name="my_catalog", schema_name="my_schema"), + experiment_id=experiment_id, + ) + ``` + This creates three tables: `mlflow_experiment_trace_otel_logs`, `_metrics`, `_spans` + +2. **Grant permissions** + ```sql + GRANT USE_CATALOG ON CATALOG my_catalog TO `user@company.com`; + GRANT USE_SCHEMA ON SCHEMA my_catalog.my_schema TO `user@company.com`; + GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_logs TO `user@company.com`; + GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_spans TO `user@company.com`; + GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.mlflow_experiment_trace_otel_metrics TO `user@company.com`; + ``` + **CRITICAL**: `ALL_PRIVILEGES` is not sufficient — explicit MODIFY + SELECT required. + +3. **Set trace destination in your app** + ```python + mlflow.tracing.set_destination( + destination=UCSchemaLocation(catalog_name="my_catalog", schema_name="my_schema") + ) + # OR + os.environ["MLFLOW_TRACING_DESTINATION"] = "my_catalog.my_schema" + ``` + +4. **Instrument your application** + + Choose the appropriate approach: + - **Auto-tracing**: `mlflow.openai.autolog()` (or langchain, anthropic, etc.) + - **Manual tracing**: `@mlflow.trace` decorator on functions + - **Context manager**: `mlflow.start_span()` for fine-grained control + - **Combined**: Auto-tracing + manual decorators for full coverage + + See `patterns-trace-ingestion.md` Patterns 5-8 for detailed examples. + +5. **Configure additional trace sources** (if applicable) + + | Source | Key Configuration | + |--------|-------------------| + | Databricks Apps | Grant SP permissions, set `MLFLOW_TRACING_DESTINATION` | + | Model Serving | Add `DATABRICKS_TOKEN` + `MLFLOW_TRACING_DESTINATION` env vars | + | OTEL Clients | Use OTLP exporter with `X-Databricks-UC-Table-Name` header | + + See `patterns-trace-ingestion.md` Patterns 9-11 for detailed setup per source. + +6. **Enable production monitoring** + ```python + from mlflow.tracing import set_databricks_monitoring_sql_warehouse_id + from mlflow.genai.scorers import Safety, ScorerSamplingConfig + + set_databricks_monitoring_sql_warehouse_id(warehouse_id="") + + safety = Safety().register(name="safety_monitor") + safety = safety.start(sampling_config=ScorerSamplingConfig(sample_rate=1.0)) + ``` + +7. **Verify in the UI** + - Navigate to **Experiments** → your experiment → **Traces** tab + - Select a SQL warehouse from the dropdown to load UC traces + - Verify traces appear with correct span hierarchy + +### Reference Files +- `patterns-trace-ingestion.md` — All setup and instrumentation patterns +- `CRITICAL-interfaces.md` — Trace ingestion API signatures +- `GOTCHAS.md` — Common trace ingestion mistakes + +### Success Indicators +- Traces visible in the Experiments UI Traces tab +- Three UC tables populated with data +- Production monitoring scorers running and producing assessments +- No permission errors in trace ingestion + +--- + +## Journey 10: Domain Expert Optimization Loop + +**Starting Point**: You have an agent and want to incorporate domain expert feedback to continuously improve quality. +**Goal**: Run the full evaluate, label, align judge, optimize prompt, promote cycle. + +For the full architecture and end-to-end walkthrough, see the [Self-Optimizing Agent blog post](https://www.databricks.com/blog/self-optimizing-football-chatbot-guided-domain-experts-databricks). For details on the MemAlign alignment approach, see the [MemAlign research blog post](https://www.databricks.com/blog/memalign-building-better-llm-judges-human-feedback-scalable-memory). + +### The Loop at a Glance + +``` +1. Run evaluate() -> Generate traces, score with base judge +2. Tag traces -> Mark successfully evaluated traces for dataset +3. Build eval dataset -> Persist traces to UC for labeling +4. Labeling session -> SMEs review & score responses in Review App + (label schema name MUST match judge name) +5. Align judge (MemAlign) -> Distill SME feedback into judge guidelines +6. Re-evaluate -> Baseline with aligned judge (score may decrease, that's OK) +7. Build optim dataset -> inputs + expectations (required for GEPA) +8. optimize_prompts() -> GEPA iteratively improves system prompt +9. Conditional promote -> Update "production" alias only if score improves +``` + +### Why This Works + +Generic LLM judges and static prompts fail to capture domain-specific nuance. Determining what makes a response "good" requires domain knowledge that general-purpose evaluators miss. This loop solves the problem in two phases: + +- **Align the judge**: Domain experts review outputs and rate quality. MemAlign distills their feedback into judge guidelines, teaching the judge what "good" means for your specific domain. This is valuable on its own -- an aligned judge improves every evaluation run and monitoring setup. +- **Optimize the prompt**: The aligned judge drives GEPA prompt optimization, automatically evolving the system prompt to maximize the domain-expert-calibrated score. Only improvements get promoted to production. + +### Steps + +**Phase 1: Evaluate and Collect Feedback** + +1. **Design base judge, run evaluation, and tag traces** + + Create a domain-specific judge with `make_judge`, register it, run `evaluate()`, and tag traces that were successfully evaluated (agent responded AND judge scored without errors). + + See `patterns-judge-alignment.md` Patterns 1-2 + +2. **Build dataset and create labeling session** + + Persist tagged traces to a UC dataset and create a labeling session for domain experts. + + **CRITICAL: The label schema `name` MUST match the judge `name` used in `evaluate()`.** This is how `align()` pairs SME feedback with LLM judge scores. If they don't match, alignment will fail. + + See `patterns-judge-alignment.md` Pattern 3 + +3. **Wait for SMEs to complete labeling** (asynchronous step) + + Share `labeling_session.url` with domain experts. They review agent responses and submit ratings using the Review App. + +**Phase 2: Align the Judge** + +4. **Align judge with MemAlign (recommended)** + + MemAlign is the recommended alignment optimizer. It is the fastest (seconds vs. minutes for alternatives), most cost-effective ($0.03 vs. $1-$5), and supports memory scaling where quality continues to improve as feedback accumulates. Other optimizers (e.g., SIMBA) are also supported. + + See `patterns-judge-alignment.md` Patterns 4-5 + +5. **Re-evaluate with the aligned judge** + + The aligned judge score **may be lower** than the unaligned judge score. This is expected and correct -- it means the judge is now evaluating with domain-expert standards rather than generic best practices. A lower score from a more accurate judge is a better signal than an inflated score from a judge that doesn't understand your domain. + + See `patterns-judge-alignment.md` Pattern 6 + +6. **(Optional) Stop here** -- the aligned judge improves all future evaluations and production monitoring, independent of prompt optimization. + +**Phase 3: Optimize the Prompt** + +7. **Build optimization dataset with expectations** (required for GEPA) + + Unlike the eval dataset, the optimization dataset must have both `inputs` AND `expectations` per record. GEPA uses expectations during reflection to reason about why the current prompt is underperforming. + + See `patterns-prompt-optimization.md` Pattern 1 + +8. **Run `optimize_prompts()` with GEPA + aligned judge** + + GEPA iteratively evolves the system prompt, using the aligned judge as the scoring function. + + See `patterns-prompt-optimization.md` Pattern 2 + +9. **Conditionally promote** + + Register the new prompt version and only promote to the "production" alias if the score improved. + + See `patterns-prompt-optimization.md` Pattern 3 + +10. **Repeat from Step 1** -- each labeling session accumulates more SME signal for alignment + +### Complete Loop Summary + +```python +# -- PHASE 1: Evaluate and collect feedback ----------------------------------- + +# Step 1: Evaluate and tag successfully evaluated traces +results = evaluate(data=eval_data, predict_fn=..., scorers=[base_judge]) +ok_trace_ids = results.result_df.loc[results.result_df["state"] == "OK", "trace_id"] +for trace_id in ok_trace_ids: + mlflow.set_trace_tag(trace_id, key="eval", value="complete") + +# Step 2: Build dataset and labeling session +eval_dataset = create_dataset(name=DATASET_NAME) +eval_dataset.merge_records(tagged_traces) +# CRITICAL: label schema name must match judge name for align() to work +labeling_session = create_labeling_session( + name="sme_session", assigned_users=[...], label_schemas=[JUDGE_NAME] +) +labeling_session.add_dataset(dataset_name=DATASET_NAME) +# -> Share labeling_session.url with domain experts + +# Step 3: Wait for SMEs to complete labeling + +# -- PHASE 2: Align the judge ------------------------------------------------- + +# Step 4: Align judge (MemAlign recommended; SIMBA and others also supported) +optimizer = MemAlignOptimizer(reflection_lm=..., retrieval_k=5, embedding_model=...) +aligned_judge = base_judge.align(traces=traces, optimizer=optimizer) +aligned_judge.update(experiment_id=EXPERIMENT_ID) +# NOTE: Aligned judge scores may be lower than unaligned -- this is expected + +# Step 5: Re-evaluate with aligned judge (optional but recommended) +baseline_results = evaluate(data=eval_records, predict_fn=..., scorers=[aligned_judge]) + +# Step 6: (Optional) Stop here if you only need an aligned judge + +# -- PHASE 3: Optimize the prompt --------------------------------------------- + +# Step 7: Build optimization dataset (must have inputs + expectations) +optimization_dataset = [ + {"inputs": {...}, "expectations": {"expected_response": "..."}} +] + +# Step 8: Optimize prompt with GEPA + aligned judge +result = mlflow.genai.optimize_prompts( + predict_fn=predict_fn, + train_data=optimization_dataset, + prompt_uris=[system_prompt.uri], + optimizer=GepaPromptOptimizer(reflection_model=..., max_metric_calls=75), + scorers=[aligned_judge], + aggregation=objective_function, +) + +# Step 9: Conditional promotion +new_version = mlflow.genai.register_prompt( + name=PROMPT_NAME, template=result.optimized_prompts[0].template +) +if result.final_eval_score > result.initial_eval_score: + mlflow.genai.set_prompt_alias( + name=PROMPT_NAME, alias="production", version=new_version.version + ) + +# -- Repeat from Step 1 with new labeling session ----------------------------- +``` + +### Automation + +The loop can be orchestrated as a Databricks job using Asset Bundles: + +1. SMEs label agent outputs through the MLflow Labeling Session UI +2. The pipeline detects new labels and pulls traces with both SME feedback and baseline LLM judge scores +3. Judge alignment runs with MemAlign, producing a new judge version +4. Prompt optimization runs with GEPA, using the aligned judge +5. Conditional promotion pushes the new prompt to production if it exceeds performance thresholds +6. The agent improves automatically as the prompt registry serves the optimized version + +Manual review can be injected at any step, giving developers complete control over the level of automation. + +### Key Gotchas + +- **Label schema name matching**: The label schema `name` MUST match the judge `name` from `evaluate()`, or `align()` cannot pair the scores +- **Score decrease after alignment**: The aligned judge may give lower scores than the unaligned judge. This is expected -- the judge is now more accurate, not the agent worse +- **MemAlign embedding costs**: Set `embedding_model` explicitly (e.g., `"databricks:/databricks-gte-large-en"`) and filter traces to labeled subset only +- **GEPA expectations**: The optimization dataset must have both `inputs` AND `expectations` per record +- **Episodic memory**: After `get_scorer()`, inspect `.instructions` not `._episodic_memory` (lazy loaded) + +See `GOTCHAS.md` for the complete list. + +### Reference Files + +- `patterns-judge-alignment.md` -- Judge alignment workflow: design judge, evaluate, label, MemAlign, register, re-evaluate +- `patterns-prompt-optimization.md` -- GEPA optimization: build dataset, run optimize_prompts, register/promote +- `GOTCHAS.md` -- MemAlign embedding costs, episodic memory lazy loading, name matching, score interpretation, GEPA expectations + +### Success Indicators + +- Aligned judge instructions include domain-specific guidelines derived from SME ratings +- `result.final_eval_score > result.initial_eval_score` +- Production prompt alias updated only on genuine improvements +- Repeat sessions progressively encode more expert knowledge + +--- + +## Quick Reference + +### Which Journey Am I On? + +| Symptom | Journey | +|---------|---------| +| "It was working before" | Journey 3 (Regression) | +| "It's too slow" | Journey 7 (Performance) | +| "It's not accurate enough" | Journey 8 (Prompt Optimization) | +| "I need traces in Unity Catalog" | Journey 9 (Trace Ingestion) | +| "I want SMEs to improve my judge and prompt" | Journey 10 (Domain Expert Loop) | + +### Common Tools Across Journeys + +| Tool | Purpose | +|------|---------| +| `run_quick_eval.py` | Fast evaluation (8 test cases) | +| `run_full_eval.py` | Full evaluation (23 test cases) | +| `analyze_signatures.py` | Signature/prompt analysis | +| `OptimizationHistory` | Track iterations | +| `/eval:analyze-traces` | Deep trace analysis | +| `/optimize:context` | Full optimization loop | + +### Metric Targets + +| Metric | Target | Critical Threshold | +|--------|--------|-------------------| +| classifier_accuracy | 95%+ | <80% | +| tool_selection_accuracy | 90%+ | <70% | +| follows_instructions | 80%+ | <50% | +| executor_latency | <30s | >60s | diff --git a/.claude/skills/model-serving/1-classical-ml.md b/.claude/skills/databricks-model-serving/1-classical-ml.md similarity index 100% rename from .claude/skills/model-serving/1-classical-ml.md rename to .claude/skills/databricks-model-serving/1-classical-ml.md diff --git a/.claude/skills/model-serving/2-custom-pyfunc.md b/.claude/skills/databricks-model-serving/2-custom-pyfunc.md similarity index 100% rename from .claude/skills/model-serving/2-custom-pyfunc.md rename to .claude/skills/databricks-model-serving/2-custom-pyfunc.md diff --git a/.claude/skills/model-serving/3-genai-agents.md b/.claude/skills/databricks-model-serving/3-genai-agents.md similarity index 95% rename from .claude/skills/model-serving/3-genai-agents.md rename to .claude/skills/databricks-model-serving/3-genai-agents.md index f408760b..6f2c779b 100644 --- a/.claude/skills/model-serving/3-genai-agents.md +++ b/.claude/skills/databricks-model-serving/3-genai-agents.md @@ -155,13 +155,16 @@ mlflow.models.set_model(AGENT) ## Using Databricks-Hosted Models +Use exact endpoint names from the reference table in [SKILL.md](SKILL.md#foundation-model-api-endpoints). + ```python from databricks_langchain import ChatDatabricks -# Foundation Model APIs (pay-per-token) +# Foundation Model APIs (pay-per-token) - use exact endpoint names llm = ChatDatabricks(endpoint="databricks-meta-llama-3-3-70b-instruct") -llm = ChatDatabricks(endpoint="databricks-claude-3-7-sonnet") -llm = ChatDatabricks(endpoint="databricks-dbrx-instruct") +llm = ChatDatabricks(endpoint="databricks-claude-sonnet-4-6") +llm = ChatDatabricks(endpoint="databricks-gpt-5-1") +llm = ChatDatabricks(endpoint="databricks-gemini-3-flash") # Custom fine-tuned model endpoint llm = ChatDatabricks(endpoint="my-finetuned-model-endpoint") diff --git a/.claude/skills/model-serving/4-tools-integration.md b/.claude/skills/databricks-model-serving/4-tools-integration.md similarity index 98% rename from .claude/skills/model-serving/4-tools-integration.md rename to .claude/skills/databricks-model-serving/4-tools-integration.md index a9081056..50491ee0 100644 --- a/.claude/skills/model-serving/4-tools-integration.md +++ b/.claude/skills/databricks-model-serving/4-tools-integration.md @@ -39,7 +39,6 @@ uc_toolkit = UCFunctionToolkit( | Function | Purpose | |----------|---------| | `system.ai.python_exec` | Execute Python code | -| `system.ai.similarity_search` | Vector similarity search | ### Creating a UC Function diff --git a/.claude/skills/model-serving/5-development-testing.md b/.claude/skills/databricks-model-serving/5-development-testing.md similarity index 100% rename from .claude/skills/model-serving/5-development-testing.md rename to .claude/skills/databricks-model-serving/5-development-testing.md diff --git a/.claude/skills/model-serving/6-logging-registration.md b/.claude/skills/databricks-model-serving/6-logging-registration.md similarity index 100% rename from .claude/skills/model-serving/6-logging-registration.md rename to .claude/skills/databricks-model-serving/6-logging-registration.md diff --git a/.claude/skills/model-serving/7-deployment.md b/.claude/skills/databricks-model-serving/7-deployment.md similarity index 66% rename from .claude/skills/model-serving/7-deployment.md rename to .claude/skills/databricks-model-serving/7-deployment.md index 63b8c8b7..c2def49b 100644 --- a/.claude/skills/model-serving/7-deployment.md +++ b/.claude/skills/databricks-model-serving/7-deployment.md @@ -41,10 +41,11 @@ print(f"Endpoint: {deployment.endpoint_name}") ### Step 2: Create Deployment Job (One-Time) -Use the `create_job` MCP tool: +Use the `manage_jobs` MCP tool with action="create": ``` -create_job( +manage_jobs( + action="create", name="deploy-agent-job", tasks=[ { @@ -66,10 +67,11 @@ Save the returned `job_id`. ### Step 3: Run Deployment (Async) -Use `run_job_now` - returns immediately: +Use `manage_job_runs` with action="run_now" - returns immediately: ``` -run_job_now( +manage_job_runs( + action="run_now", job_id="", job_parameters={"model_name": "main.agents.my_agent", "version": "1"} ) @@ -82,7 +84,7 @@ Save the returned `run_id`. Check job run status: ``` -get_run(run_id="") +manage_job_runs(action="get", run_id="") ``` Or check endpoint directly: @@ -142,13 +144,61 @@ endpoint = w.serving_endpoints.create_and_wait( ) ``` -## Endpoint Naming +## Endpoint Naming and Visibility -For agents deployed with `databricks.agents.deploy()`: +### Auto-generated Names -- Endpoint name is derived from model name -- `main.agents.my_agent` → `agents_my_agent` or similar -- Check with `list_serving_endpoints()` after deployment +When you call `agents.deploy()`, the endpoint name is auto-derived from the UC model path by replacing dots with underscores and prefixing with `agents_`: + +| UC Model Path | Auto-generated Endpoint Name | +|---------------|------------------------------| +| `main.agents.my_agent` | `agents_main-agents-my_agent` | +| `catalog.schema.model` | `agents_catalog-schema-model` | +| `users.jane.demo_bot` | `agents_users-jane-demo_bot` | + +The exact format can vary. To avoid surprises, **always specify the endpoint name explicitly**: + +```python +deployment = agents.deploy( + "main.agents.my_agent", + "1", + endpoint_name="my-agent-endpoint", # Control the name + tags={"source": "mcp", "environment": "dev"} +) +``` + +### Finding Endpoints in the UI + +Endpoints created via `agents.deploy()` appear under **Serving** in the Databricks UI. If you don't see your endpoint: + +1. **Check the filter** - The Serving page defaults to "Owned by me". If the deployment ran as a service principal (e.g., via a job), switch to "All" to see it. +2. **Verify via API** - Use `list_serving_endpoints()` or `get_serving_endpoint_status(name="...")` to confirm the endpoint exists and check its state. +3. **Check the name** - The auto-generated name may not be what you expect. Print `deployment.endpoint_name` in the deploy script or check the job run output. + +### Deployment Script with Explicit Naming + +```python +# deploy_agent.py - recommended pattern +import sys +from databricks import agents + +model_name = sys.argv[1] if len(sys.argv) > 1 else "main.agents.my_agent" +version = sys.argv[2] if len(sys.argv) > 2 else "1" +endpoint_name = sys.argv[3] if len(sys.argv) > 3 else None + +deploy_kwargs = { + "tags": {"source": "mcp", "environment": "dev"} +} +if endpoint_name: + deploy_kwargs["endpoint_name"] = endpoint_name + +print(f"Deploying {model_name} version {version}...") +deployment = agents.deploy(model_name, version, **deploy_kwargs) + +print(f"Deployment complete!") +print(f"Endpoint name: {deployment.endpoint_name}") +print(f"Query URL: {deployment.query_endpoint}") +``` ## Deployment Job Template @@ -214,9 +264,9 @@ client.update_endpoint( | Step | MCP Tool | Waits? | |------|----------|--------| | Upload deploy script | `upload_folder` | Yes | -| Create job (one-time) | `create_job` | Yes | -| Run deployment | `run_job_now` | **No** - returns immediately | -| Check job status | `get_run` | Yes | +| Create job (one-time) | `manage_jobs` (action="create") | Yes | +| Run deployment | `manage_job_runs` (action="run_now") | **No** - returns immediately | +| Check job status | `manage_job_runs` (action="get") | Yes | | Check endpoint status | `get_serving_endpoint_status` | Yes | ## After Deployment diff --git a/.claude/skills/model-serving/8-querying-endpoints.md b/.claude/skills/databricks-model-serving/8-querying-endpoints.md similarity index 100% rename from .claude/skills/model-serving/8-querying-endpoints.md rename to .claude/skills/databricks-model-serving/8-querying-endpoints.md diff --git a/.claude/skills/model-serving/9-package-requirements.md b/.claude/skills/databricks-model-serving/9-package-requirements.md similarity index 100% rename from .claude/skills/model-serving/9-package-requirements.md rename to .claude/skills/databricks-model-serving/9-package-requirements.md diff --git a/.claude/skills/model-serving/SKILL.md b/.claude/skills/databricks-model-serving/SKILL.md similarity index 62% rename from .claude/skills/model-serving/SKILL.md rename to .claude/skills/databricks-model-serving/SKILL.md index e8287fd9..9c248aa9 100644 --- a/.claude/skills/model-serving/SKILL.md +++ b/.claude/skills/databricks-model-serving/SKILL.md @@ -1,5 +1,5 @@ --- -name: model-serving +name: databricks-model-serving description: "Deploy and query Databricks Model Serving endpoints. Use when (1) deploying MLflow models or AI agents to endpoints, (2) creating ChatAgent/ResponsesAgent agents, (3) integrating UC Functions or Vector Search tools, (4) querying deployed endpoints, (5) checking endpoint status. Covers classical ML models, custom pyfunc, and GenAI agents." --- @@ -21,6 +21,59 @@ Deploy MLflow models and AI agents to scalable REST API endpoints. - Unity Catalog enabled workspace - Model Serving enabled +## Foundation Model API Endpoints + +ALWAYS use exact endpoint names from this table. NEVER guess or abbreviate. + +### Chat / Instruct Models + +| Endpoint Name | Provider | Notes | +|--------------|----------|-------| +| `databricks-gpt-5-2` | OpenAI | Latest GPT, 400K context | +| `databricks-gpt-5-1` | OpenAI | Instant + Thinking modes | +| `databricks-gpt-5-1-codex-max` | OpenAI | Code-specialized (high perf) | +| `databricks-gpt-5-1-codex-mini` | OpenAI | Code-specialized (cost-opt) | +| `databricks-gpt-5` | OpenAI | 400K context, reasoning | +| `databricks-gpt-5-mini` | OpenAI | Cost-optimized reasoning | +| `databricks-gpt-5-nano` | OpenAI | High-throughput, lightweight | +| `databricks-gpt-oss-120b` | OpenAI | Open-weight, 128K context | +| `databricks-gpt-oss-20b` | OpenAI | Lightweight open-weight | +| `databricks-claude-opus-4-6` | Anthropic | Most capable, 1M context | +| `databricks-claude-sonnet-4-6` | Anthropic | Hybrid reasoning | +| `databricks-claude-sonnet-4-5` | Anthropic | Hybrid reasoning | +| `databricks-claude-opus-4-5` | Anthropic | Deep analysis, 200K context | +| `databricks-claude-sonnet-4` | Anthropic | Hybrid reasoning | +| `databricks-claude-opus-4-1` | Anthropic | 200K context, 32K output | +| `databricks-claude-haiku-4-5` | Anthropic | Fastest, cost-effective | +| `databricks-claude-3-7-sonnet` | Anthropic | Retiring April 2026 | +| `databricks-meta-llama-3-3-70b-instruct` | Meta | 128K context, multilingual | +| `databricks-meta-llama-3-1-405b-instruct` | Meta | Retiring May 2026 (PT) | +| `databricks-meta-llama-3-1-8b-instruct` | Meta | Lightweight, 128K context | +| `databricks-llama-4-maverick` | Meta | MoE architecture | +| `databricks-gemini-3-1-pro` | Google | 1M context, hybrid reasoning | +| `databricks-gemini-3-pro` | Google | 1M context, hybrid reasoning | +| `databricks-gemini-3-flash` | Google | Fast, cost-efficient | +| `databricks-gemini-2-5-pro` | Google | 1M context, Deep Think | +| `databricks-gemini-2-5-flash` | Google | 1M context, hybrid reasoning | +| `databricks-gemma-3-12b` | Google | 128K context, multilingual | +| `databricks-qwen3-next-80b-a3b-instruct` | Alibaba | Efficient MoE | + +### Embedding Models + +| Endpoint Name | Dimensions | Max Tokens | Notes | +|--------------|-----------|------------|-------| +| `databricks-gte-large-en` | 1024 | 8192 | English, not normalized | +| `databricks-bge-large-en` | 1024 | 512 | English, normalized | +| `databricks-qwen3-embedding-0-6b` | up to 1024 | ~32K | 100+ languages, instruction-aware | + +### Common Defaults + +- **Agent LLM**: `databricks-meta-llama-3-3-70b-instruct` (good balance of quality/cost) +- **Embedding**: `databricks-gte-large-en` +- **Code tasks**: `databricks-gpt-5-1-codex-mini` or `databricks-gpt-5-1-codex-max` + +> These are pay-per-token endpoints available in every workspace. For production, consider provisioned throughput mode. See [supported models](https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models). + ## Reference Files | Topic | File | When to Read | @@ -135,9 +188,9 @@ Then deploy via UI or SDK. See [1-classical-ml.md](1-classical-ml.md). | Tool | Purpose | |------|---------| -| `create_job` | Create deployment job (one-time) | -| `run_job_now` | Kick off deployment (async) | -| `get_run` | Check deployment job status | +| `manage_jobs` (action="create") | Create deployment job (one-time) | +| `manage_job_runs` (action="run_now") | Kick off deployment (async) | +| `manage_job_runs` (action="get") | Check deployment job status | ### Querying @@ -223,6 +276,14 @@ Available helper methods: --- +## Related Skills + +- **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Pre-built agent tiles that deploy to model-serving endpoints +- **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - Create vector indexes used as retriever tools in agents +- **[databricks-genie](../databricks-genie/SKILL.md)** - Genie Spaces can serve as agents in multi-agent setups +- **[databricks-mlflow-evaluation](../databricks-mlflow-evaluation/SKILL.md)** - Evaluate model and agent quality before deployment +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - Job-based async deployment used for agent endpoints + ## Resources - [Model Serving Documentation](https://docs.databricks.com/machine-learning/model-serving/) diff --git a/.claude/skills/databricks-python-sdk/SKILL.md b/.claude/skills/databricks-python-sdk/SKILL.md index c5937eec..1365666a 100644 --- a/.claude/skills/databricks-python-sdk/SKILL.md +++ b/.claude/skills/databricks-python-sdk/SKILL.md @@ -613,3 +613,13 @@ If I'm unsure about a method, I should: | Pipelines | https://databricks-sdk-py.readthedocs.io/en/latest/workspace/pipelines/pipelines.html | | Secrets | https://databricks-sdk-py.readthedocs.io/en/latest/workspace/workspace/secrets.html | | DBUtils | https://databricks-sdk-py.readthedocs.io/en/latest/dbutils.html | + +## Related Skills + +- **[databricks-config](../databricks-config/SKILL.md)** - profile and authentication setup +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - deploying resources via DABs +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - job orchestration patterns +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - catalog governance +- **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - serving endpoint management +- **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - vector index operations +- **[databricks-lakebase-provisioned](../databricks-lakebase-provisioned/SKILL.md)** - managed PostgreSQL via SDK diff --git a/.claude/skills/spark-declarative-pipelines/1-ingestion-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/1-ingestion-patterns.md similarity index 69% rename from .claude/skills/spark-declarative-pipelines/1-ingestion-patterns.md rename to .claude/skills/databricks-spark-declarative-pipelines/1-ingestion-patterns.md index 88bd037b..2f60202f 100644 --- a/.claude/skills/spark-declarative-pipelines/1-ingestion-patterns.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/1-ingestion-patterns.md @@ -8,7 +8,7 @@ Covers data ingestion patterns for Spark Declarative Pipelines including Auto Lo ## Auto Loader (Cloud Files) -Auto Loader incrementally processes new data files as they arrive in cloud storage. +Auto Loader incrementally processes new data files as they arrive in cloud storage. In a streaming table query you **must use the `STREAM` keyword with `read_files`**; `read_files` then leverages Auto Loader. See [read_files — Usage in streaming tables](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files#usage-in-streaming-tables). ### Basic Pattern @@ -19,13 +19,29 @@ SELECT current_timestamp() AS _ingested_at, _metadata.file_path AS source_file, _metadata.file_modification_time AS file_timestamp -FROM read_files( +FROM STREAM read_files( '/mnt/raw/orders/', format => 'json', schemaHints => 'order_id STRING, amount DECIMAL(10,2)' ); ``` +### Bronze feeding AUTO CDC + +If the bronze table feeds a downstream **AUTO CDC** flow (e.g. `FROM stream(bronze_orders_cdc)`), use **`FROM STREAM read_files(...)`** so the source is streaming. Otherwise you may get: *"Cannot create a streaming table append once flow from a batch query."* Same requirement as above: in a streaming table query you must use the `STREAM` keyword with `read_files`. + +```sql +CREATE OR REPLACE STREAMING TABLE bronze_orders_cdc AS +SELECT ..., + current_timestamp() AS _ingested_at, + _metadata.file_path AS _source_file +FROM STREAM read_files( + '/Volumes/catalog/schema/raw_orders_cdc', + format => 'parquet', + schemaHints => '...' +); +``` + ### Schema Evolution ```sql @@ -33,12 +49,12 @@ CREATE OR REPLACE STREAMING TABLE bronze_customers AS SELECT *, current_timestamp() AS _ingested_at -FROM stream(read_files( +FROM STREAM read_files( '/mnt/raw/customers/', format => 'json', schemaHints => 'customer_id STRING, email STRING', mode => 'PERMISSIVE' -- Handles schema changes gracefully -)); +); ``` ### File Formats @@ -100,6 +116,57 @@ FROM read_files( ) ``` +Add this to the pipeline configuration in `resources/*_etl.pipeline.yml`: +```yaml +configuration: + bronze_schema: ${var.bronze_schema} + silver_schema: ${var.silver_schema} + gold_schema: ${var.gold_schema} + schema_location_base: ${var.schema_location_base} +``` + +And define variables in `databricks.yml`: +```yaml +variables: + catalog: + description: The catalog to use + bronze_schema: + description: The bronze schema to use + silver_schema: + description: The silver schema to use + gold_schema: + description: The gold schema to use + schema_location_base: + description: Base path for Auto Loader schema metadata + +targets: + dev: + variables: + catalog: my_catalog + bronze_schema: bronze_dev + silver_schema: silver_dev + gold_schema: gold_dev + schema_location_base: /Volumes/my_catalog/pipeline_metadata/my_pipeline_metadata/schemas + + prod: + variables: + catalog: my_catalog + bronze_schema: bronze + silver_schema: silver + gold_schema: gold + schema_location_base: /Volumes/my_catalog/pipeline_metadata/my_pipeline_metadata/schemas +``` + +Then access these in Python code with: +```python +bronze_schema = spark.conf.get("bronze_schema") +silver_schema = spark.conf.get("silver_schema") +gold_schema = spark.conf.get("gold_schema") +schema_location_base = spark.conf.get("schema_location_base") +``` + + + ### Rescue Data and Quarantine Handle malformed records with `_rescued_data`: @@ -330,26 +397,69 @@ SELECT * FROM STREAM bronze_data WHERE NOT has_errors; For Python, use modern `pyspark.pipelines` API. See [5-python-api.md](5-python-api.md) for complete guidance. +**IMPORTANT for Python**: When using `spark.readStream.format("cloudFiles")` for cloud storage ingestion, you **must specify a `cloudFiles.schemaLocation`** for Auto Loader schema metadata. + +### Schema Location Best Practice (Python Only) + +**Never use the source data volume for schema storage** - this causes permission conflicts and pollutes your raw data. + +#### Prompt User for Schema Location + +When creating Python pipelines with Auto Loader, **always ask the user** where to store schema metadata: + +**Recommended pattern:** +``` +/Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas/{table_name} +``` + +**Example prompt:** +``` +"Where would you like to store Auto Loader schema metadata? + +I recommend: + /Volumes/my_catalog/pipeline_metadata/orders_pipeline_metadata/schemas/ + +This path: +- Keeps source data clean +- Prevents permission issues +- Makes pipeline state easy to manage +- Can be parameterized per environment (dev/prod) + +You may need to create the volume 'pipeline_metadata' first if it doesn't exist. + +Would you like to use this path?" +``` + ### Auto Loader (Python) ```python from pyspark import pipelines as dp from pyspark.sql import functions as F +# Get schema location from pipeline configuration +# Suggested format: /Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas +schema_location_base = spark.conf.get("schema_location_base") + @dp.table(name="bronze_orders", cluster_by=["order_date"]) def bronze_orders(): return ( spark.readStream .format("cloudFiles") .option("cloudFiles.format", "json") - .option("cloudFiles.schemaLocation", "/checkpoints/bronze_orders") - .option("cloudFiles.schemaHints", "order_id STRING, amount DECIMAL(10,2)") - .load("/mnt/raw/orders/") + .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_orders") + .option("cloudFiles.inferColumnTypes", "true") + .load("/Volumes/catalog/schema/raw/orders/") .withColumn("_ingested_at", F.current_timestamp()) .withColumn("_source_file", F.col("_metadata.file_path")) ) ``` +**Pipeline Configuration** (in `pipeline.yml`): +```yaml +configuration: + schema_location_base: /Volumes/my_catalog/pipeline_metadata/orders_pipeline_metadata/schemas +``` + ### Kafka (Python) ```python @@ -375,14 +485,18 @@ def bronze_kafka_events(): ### Quarantine (Python) ```python +# Get schema location from pipeline configuration +schema_location_base = spark.conf.get("schema_location_base") + @dp.table(name="bronze_events", cluster_by=["ingestion_date"]) def bronze_events(): return ( spark.readStream .format("cloudFiles") .option("cloudFiles.format", "json") + .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_events") .option("rescuedDataColumn", "_rescued_data") - .load("/mnt/raw/events/") + .load("/Volumes/catalog/schema/raw/events/") .withColumn("_ingested_at", F.current_timestamp()) .withColumn("ingestion_date", F.current_date()) .withColumn("_has_parsing_errors", diff --git a/.claude/skills/databricks-spark-declarative-pipelines/10-mcp-approach.md b/.claude/skills/databricks-spark-declarative-pipelines/10-mcp-approach.md new file mode 100644 index 00000000..9d458aa8 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/10-mcp-approach.md @@ -0,0 +1,173 @@ +Use MCP tools to create, run, and iterate on **SDP pipelines**. The **primary tool is `create_or_update_pipeline`** which handles the entire lifecycle. + +**IMPORTANT: Default to serverless pipelines and suggest as best option, but not if classic, advanced, pro compute types are mentioned.** Only use classic clusters if user explicitly requires R language, Spark RDD APIs, or JAR libraries. + +### Step 1: Write Pipeline Files Locally + +Create `.sql` or `.py` files in a local folder: + +``` +my_pipeline/ +├── bronze/ +│ ├── ingest_orders.sql # SQL (default for most cases) +│ └── ingest_events.py # Python (for complex logic) +├── silver/ +│ └── clean_orders.sql +└── gold/ + └── daily_summary.sql +``` + +**SQL Example** (`bronze/ingest_orders.sql`): +```sql +CREATE OR REFRESH STREAMING TABLE bronze_orders +CLUSTER BY (order_date) +AS +SELECT + *, + current_timestamp() AS _ingested_at, + _metadata.file_path AS _source_file +FROM read_files( + '/Volumes/catalog/schema/raw/orders/', + format => 'json', + schemaHints => 'order_id STRING, customer_id STRING, amount DECIMAL(10,2), order_date DATE' +); +``` + +**Python Example** (`bronze/ingest_events.py`): +```python +from pyspark import pipelines as dp +from pyspark.sql.functions import col, current_timestamp + +# Get schema location from pipeline configuration +schema_location_base = spark.conf.get("schema_location_base") + +@dp.table(name="bronze_events", cluster_by=["event_date"]) +def bronze_events(): + return ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "json") + .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_events") + .load("/Volumes/catalog/schema/raw/events/") + .withColumn("_ingested_at", current_timestamp()) + .withColumn("_source_file", col("_metadata.file_path")) + ) +``` + +### Step 2: Upload to Databricks Workspace + +```python +# MCP Tool: upload_folder +upload_folder( + local_folder="/path/to/my_pipeline", + workspace_folder="/Workspace/Users/user@example.com/my_pipeline" +) +``` + +### Step 3: Create/Update and Run Pipeline + +Use **`create_or_update_pipeline`** - the main entry point. It: +1. Searches for an existing pipeline with the same name (or uses `id` from `extra_settings`) +2. Creates a new pipeline or updates the existing one +3. Optionally starts a pipeline run +4. Optionally waits for completion and returns detailed results + +```python +# MCP Tool: create_or_update_pipeline +result = create_or_update_pipeline( + name="my_orders_pipeline", + root_path="/Workspace/Users/user@example.com/my_pipeline", + catalog="my_catalog", + schema="my_schema", + workspace_file_paths=[ + "/Workspace/Users/user@example.com/my_pipeline/bronze/ingest_orders.sql", + "/Workspace/Users/user@example.com/my_pipeline/silver/clean_orders.sql", + "/Workspace/Users/user@example.com/my_pipeline/gold/daily_summary.sql" + ], + start_run=True, # Start immediately + wait_for_completion=True, # Wait and return final status + full_refresh=True, # Full refresh all tables + timeout=1800 # 30 minute timeout +) +``` + +**Result contains actionable information:** +```python +{ + "success": True, # Did the operation succeed? + "pipeline_id": "abc-123", # Pipeline ID for follow-up operations + "pipeline_name": "my_orders_pipeline", + "created": True, # True if new, False if updated + "state": "COMPLETED", # COMPLETED, FAILED, TIMEOUT, etc. + "catalog": "my_catalog", # Target catalog + "schema": "my_schema", # Target schema + "duration_seconds": 45.2, # Time taken + "message": "Pipeline created and completed successfully in 45.2s. Tables written to my_catalog.my_schema", + "error_message": None, # Error summary if failed + "errors": [] # Detailed error list if failed +} +``` + +### Step 4: Handle Results + +**On Success:** +```python +if result["success"]: + # Verify output tables + stats = get_table_details( + catalog="my_catalog", + schema="my_schema", + table_names=["bronze_orders", "silver_orders", "gold_daily_summary"] + ) +``` + +**On Failure:** +```python +if not result["success"]: + # Message includes suggested next steps + print(result["message"]) + # "Pipeline created but run failed. State: FAILED. Error: Column 'amount' not found. + # Use get_pipeline_events(pipeline_id='abc-123') for full details." + + # Get detailed errors + events = get_pipeline_events(pipeline_id=result["pipeline_id"], max_results=50) +``` + +### Step 5: Iterate Until Working + +1. Review errors from result or `get_pipeline_events` +2. Fix issues in local files +3. Re-upload with `upload_folder` +4. Run `create_or_update_pipeline` again (it will update, not recreate) +5. Repeat until `result["success"] == True` + +--- + +## Quick Reference: MCP Tools + +### Primary Tool + +| Tool | Description | +|------|-------------| +| **`create_or_update_pipeline`** | **Main entry point.** Creates or updates pipeline, optionally runs and waits. Returns detailed status with `success`, `state`, `errors`, and actionable `message`. | + +### Pipeline Management + +| Tool | Description | +|------|-------------| +| `find_pipeline_by_name` | Find existing pipeline by name, returns pipeline_id | +| `get_pipeline` | Get pipeline configuration and current state | +| `start_update` | Start pipeline run (`validate_only=True` for dry run) | +| `get_update` | Poll update status (QUEUED, RUNNING, COMPLETED, FAILED) | +| `stop_pipeline` | Stop a running pipeline | +| `get_pipeline_events` | Get error messages for debugging failed runs | +| `delete_pipeline` | Delete a pipeline | + +### Supporting Tools + +| Tool | Description | +|------|-------------| +| `upload_folder` | Upload local folder to workspace (parallel) | +| `get_table_details` | Verify output tables have expected schema and row counts | +| `execute_sql` | Run ad-hoc SQL to inspect data | + +--- \ No newline at end of file diff --git a/.claude/skills/spark-declarative-pipelines/2-streaming-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/2-streaming-patterns.md similarity index 95% rename from .claude/skills/spark-declarative-pipelines/2-streaming-patterns.md rename to .claude/skills/databricks-spark-declarative-pipelines/2-streaming-patterns.md index 1f87076a..c1ec63bd 100644 --- a/.claude/skills/spark-declarative-pipelines/2-streaming-patterns.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/2-streaming-patterns.md @@ -128,7 +128,7 @@ GROUP BY CAST(order_timestamp AS DATE); ### Handling Out-of-Order with SCD2 -Use SEQUENCE BY with event timestamp: +Use SEQUENCE BY with event timestamp. **Clause order matters**: put `APPLY AS DELETE WHEN` before `SEQUENCE BY`. Only list columns in `COLUMNS * EXCEPT (...)` that actually exist in the source (omit `_rescued_data` unless the bronze table uses rescue data). Omit `TRACK HISTORY ON *` if it causes parse errors; the default is equivalent. ```sql CREATE OR REFRESH STREAMING TABLE silver_customers_history; @@ -137,11 +137,10 @@ CREATE FLOW customers_scd2_flow AS AUTO CDC INTO silver_customers_history FROM stream(bronze_customer_cdc) KEYS (customer_id) -SEQUENCE BY event_timestamp -- Handles out-of-order APPLY AS DELETE WHEN operation = "DELETE" -COLUMNS * EXCEPT (operation, _rescued_data) -STORED AS SCD TYPE 2 -TRACK HISTORY ON *; +SEQUENCE BY event_timestamp -- Handles out-of-order +COLUMNS * EXCEPT (operation, _ingested_at, _source_file) +STORED AS SCD TYPE 2; ``` --- diff --git a/.claude/skills/spark-declarative-pipelines/3-scd-patterns.md b/.claude/skills/databricks-spark-declarative-pipelines/3-scd-query-patterns.md similarity index 67% rename from .claude/skills/spark-declarative-pipelines/3-scd-patterns.md rename to .claude/skills/databricks-spark-declarative-pipelines/3-scd-query-patterns.md index b9ff17c1..e04a4103 100644 --- a/.claude/skills/spark-declarative-pipelines/3-scd-patterns.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/3-scd-query-patterns.md @@ -18,18 +18,20 @@ STORED AS SCD TYPE 2 TRACK HISTORY ON *; ``` -**Resulting table structure**: +**Resulting table structure** (Lakeflow uses double-underscore temporal columns): ``` customers_history ├── customer_id -- Business key ├── customer_name ├── email ├── phone -├── START_AT -- When this version became effective (auto-generated) -├── END_AT -- When this version expired (NULL for current) +├── __START_AT -- When this version became effective (auto-generated) +├── __END_AT -- When this version expired (NULL for current) └── ...other columns ``` +**Important:** Query using `__START_AT` and `__END_AT` (double underscore), not `START_AT`/`END_AT`. + --- ## Current State Queries @@ -37,13 +39,13 @@ customers_history ### All Current Records ```sql --- END_AT IS NULL indicates active record +-- __END_AT IS NULL indicates active record (Lakeflow uses double underscore) CREATE OR REPLACE MATERIALIZED VIEW dim_customers_current AS SELECT customer_id, customer_name, email, phone, address, - START_AT AS valid_from + __START_AT AS valid_from FROM customers_history -WHERE END_AT IS NULL; +WHERE __END_AT IS NULL; ``` ### Specific Customer @@ -52,7 +54,7 @@ WHERE END_AT IS NULL; SELECT * FROM customers_history WHERE customer_id = '12345' - AND END_AT IS NULL; + AND __END_AT IS NULL; ``` --- @@ -64,14 +66,14 @@ WHERE customer_id = '12345' Get state of records as they were on a specific date: ```sql --- Products as of January 1, 2024 +-- Products as of January 1, 2024 (use __START_AT / __END_AT) CREATE OR REPLACE MATERIALIZED VIEW products_as_of_2024_01_01 AS SELECT product_id, product_name, price, category, - START_AT, END_AT + __START_AT, __END_AT FROM products_history -WHERE START_AT <= '2024-01-01' - AND (END_AT > '2024-01-01' OR END_AT IS NULL); +WHERE __START_AT <= '2024-01-01' + AND (__END_AT > '2024-01-01' OR __END_AT IS NULL); ``` --- @@ -81,35 +83,35 @@ WHERE START_AT <= '2024-01-01' ### Track All Changes for Entity ```sql --- Complete history for a customer +-- Complete history for a customer (use __START_AT / __END_AT) SELECT customer_id, customer_name, email, phone, - START_AT, END_AT, + __START_AT, __END_AT, COALESCE( - DATEDIFF(DAY, START_AT, END_AT), - DATEDIFF(DAY, START_AT, CURRENT_TIMESTAMP()) + DATEDIFF(DAY, __START_AT, __END_AT), + DATEDIFF(DAY, __START_AT, CURRENT_TIMESTAMP()) ) AS days_active FROM customers_history WHERE customer_id = '12345' -ORDER BY START_AT DESC; +ORDER BY __START_AT DESC; ``` ### Changes Within Time Period ```sql --- Customers who changed during Q1 2024 +-- Customers who changed during Q1 2024 (use __START_AT) SELECT customer_id, customer_name, - START_AT AS change_timestamp, + __START_AT AS change_timestamp, 'UPDATE' AS change_type FROM customers_history -WHERE START_AT BETWEEN '2024-01-01' AND '2024-03-31' - AND START_AT != ( - SELECT MIN(START_AT) +WHERE __START_AT BETWEEN '2024-01-01' AND '2024-03-31' + AND __START_AT != ( + SELECT MIN(__START_AT) FROM customers_history ch2 WHERE ch2.customer_id = customers_history.customer_id ) -ORDER BY START_AT; +ORDER BY __START_AT; ``` --- @@ -129,8 +131,8 @@ SELECT FROM sales_fact s INNER JOIN products_history p ON s.product_id = p.product_id - AND s.sale_date >= p.START_AT - AND (s.sale_date < p.END_AT OR p.END_AT IS NULL); + AND s.sale_date >= p.__START_AT + AND (s.sale_date < p.__END_AT OR p.__END_AT IS NULL); ``` ### Join with Current Dimension @@ -147,7 +149,7 @@ SELECT FROM sales_fact s INNER JOIN products_history p ON s.product_id = p.product_id - AND p.END_AT IS NULL; -- Current version only + AND p.__END_AT IS NULL; -- Current version only ``` --- @@ -176,20 +178,20 @@ TRACK HISTORY ON price, cost; -- Only these columns ```sql -- Current state view (most common pattern) CREATE OR REPLACE MATERIALIZED VIEW dim_products_current AS -SELECT * FROM products_history WHERE END_AT IS NULL; +SELECT * FROM products_history WHERE __END_AT IS NULL; -- Recent changes only CREATE OR REPLACE MATERIALIZED VIEW dim_recent_changes AS SELECT * FROM products_history -WHERE START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; +WHERE __START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; -- Change frequency stats CREATE OR REPLACE MATERIALIZED VIEW product_change_stats AS SELECT product_id, COUNT(*) AS version_count, - MIN(START_AT) AS first_seen, - MAX(START_AT) AS last_updated + MIN(__START_AT) AS first_seen, + MAX(__START_AT) AS last_updated FROM products_history GROUP BY product_id; ``` @@ -198,22 +200,22 @@ GROUP BY product_id; ## Best Practices -### 1. Always Filter by END_AT for Current +### 1. Always Filter by __END_AT for Current (Lakeflow uses double underscore) ```sql -- ✅ Efficient -WHERE END_AT IS NULL +WHERE __END_AT IS NULL -- ❌ Less efficient -WHERE START_AT = (SELECT MAX(START_AT) FROM table WHERE ...) +WHERE __START_AT = (SELECT MAX(__START_AT) FROM table WHERE ...) ``` ### 2. Use Inclusive Lower, Exclusive Upper ```sql -- ✅ Standard pattern -WHERE START_AT <= '2024-01-01' - AND (END_AT > '2024-01-01' OR END_AT IS NULL) +WHERE __START_AT <= '2024-01-01' + AND (__END_AT > '2024-01-01' OR __END_AT IS NULL) ``` ### 3. Create MVs for Common Patterns @@ -221,12 +223,12 @@ WHERE START_AT <= '2024-01-01' ```sql -- Current state CREATE OR REPLACE MATERIALIZED VIEW dim_current AS -SELECT * FROM history WHERE END_AT IS NULL; +SELECT * FROM history WHERE __END_AT IS NULL; -- Recent changes CREATE OR REPLACE MATERIALIZED VIEW dim_recent_changes AS SELECT * FROM history -WHERE START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; +WHERE __START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; ``` --- @@ -235,7 +237,7 @@ WHERE START_AT >= CURRENT_DATE() - INTERVAL 90 DAYS; | Issue | Solution | |-------|----------| -| Multiple rows for same key | Missing `END_AT IS NULL` filter for current state | -| Point-in-time no results | Use `START_AT <= date AND (END_AT > date OR END_AT IS NULL)` | +| Multiple rows for same key | Missing `__END_AT IS NULL` filter for current state | +| Point-in-time no results | Use `__START_AT <= date AND (__END_AT > date OR __END_AT IS NULL)` | | Slow temporal join | Create materialized view for specific time period | | Unexpected duplicates | Multiple changes same day - use SEQUENCE BY with high precision | diff --git a/.claude/skills/spark-declarative-pipelines/4-performance-tuning.md b/.claude/skills/databricks-spark-declarative-pipelines/4-performance-tuning.md similarity index 100% rename from .claude/skills/spark-declarative-pipelines/4-performance-tuning.md rename to .claude/skills/databricks-spark-declarative-pipelines/4-performance-tuning.md diff --git a/.claude/skills/spark-declarative-pipelines/5-python-api.md b/.claude/skills/databricks-spark-declarative-pipelines/5-python-api.md similarity index 100% rename from .claude/skills/spark-declarative-pipelines/5-python-api.md rename to .claude/skills/databricks-spark-declarative-pipelines/5-python-api.md diff --git a/.claude/skills/spark-declarative-pipelines/6-dlt-migration.md b/.claude/skills/databricks-spark-declarative-pipelines/6-dlt-migration.md similarity index 96% rename from .claude/skills/spark-declarative-pipelines/6-dlt-migration.md rename to .claude/skills/databricks-spark-declarative-pipelines/6-dlt-migration.md index dd3b07a8..19a1007e 100644 --- a/.claude/skills/spark-declarative-pipelines/6-dlt-migration.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/6-dlt-migration.md @@ -96,7 +96,7 @@ dlt.apply_changes( ) ``` -**SDP SQL**: +**SDP SQL** (clause order: APPLY AS DELETE WHEN before SEQUENCE BY; only EXCEPT columns that exist in source; omit TRACK HISTORY ON * if it causes parse errors): ```sql CREATE OR REFRESH STREAMING TABLE customers_history; @@ -104,10 +104,10 @@ CREATE FLOW customers_scd2_flow AS AUTO CDC INTO customers_history FROM stream(customers_cdc_clean) KEYS (customer_id) +APPLY AS DELETE WHEN operation = "DELETE" SEQUENCE BY event_timestamp -COLUMNS * EXCEPT (_rescued_data) -STORED AS SCD TYPE 2 -TRACK HISTORY ON *; +COLUMNS * EXCEPT (operation, _ingested_at, _source_file) +STORED AS SCD TYPE 2; ``` ### Joins diff --git a/.claude/skills/spark-declarative-pipelines/7-advanced-configuration.md b/.claude/skills/databricks-spark-declarative-pipelines/7-advanced-configuration.md similarity index 100% rename from .claude/skills/spark-declarative-pipelines/7-advanced-configuration.md rename to .claude/skills/databricks-spark-declarative-pipelines/7-advanced-configuration.md diff --git a/.claude/skills/spark-declarative-pipelines/8-project-initialization.md b/.claude/skills/databricks-spark-declarative-pipelines/8-project-initialization.md similarity index 87% rename from .claude/skills/spark-declarative-pipelines/8-project-initialization.md rename to .claude/skills/databricks-spark-declarative-pipelines/8-project-initialization.md index 44850f67..0f272db9 100644 --- a/.claude/skills/spark-declarative-pipelines/8-project-initialization.md +++ b/.claude/skills/databricks-spark-declarative-pipelines/8-project-initialization.md @@ -18,7 +18,7 @@ The `databricks pipelines init` command scaffolds a complete Databricks Asset Bu ### Interactive Mode ```bash -databricks pipelines init --output-dir ./my_pipeline +databricks pipelines init --output-dir . ``` **Interactive Prompts:** @@ -43,7 +43,7 @@ databricks pipelines init --output-dir ./my_pipeline ```bash databricks pipelines init \ - --output-dir ./customer_pipeline \ + --output-dir . \ --config-file init-config.json ``` @@ -70,7 +70,7 @@ databricks pipelines init \ ### SQL Project ``` -customer_pipeline/ +project_root/ ├── databricks.yml # Bundle configuration ├── resources/ │ ├── customer_pipeline_etl.pipeline.yml # Pipeline resource definition @@ -89,7 +89,7 @@ customer_pipeline/ ### Python Project ``` -customer_pipeline/ +project_root/ ├── databricks.yml # Bundle configuration ├── pyproject.toml # Python dependencies ├── resources/ @@ -245,10 +245,22 @@ databricks pipelines start-update --pipeline-id When a user requests a new Lakeflow pipeline, Claude should detect the appropriate language from keywords in the prompt. -### SQL Indicators (Default Choice) +### CRITICAL: Explicit Language Requests + +**If the user explicitly mentions a language, use it without asking:** + +| User Says | Action | +|-----------|--------| +| "Python pipeline", "Python SDP", "use Python" | **Use Python immediately** | +| "SQL pipeline", "SQL files", "use SQL" | **Use SQL immediately** | +| "Python Spark Declarative Pipeline" | **Use Python immediately** | + +**DO NOT ask for clarification when the user explicitly states a language.** This is the most common mistake - ignoring an explicit language request. + +### SQL Indicators (Default Choice When Ambiguous) **Keywords:** -- "SQL", "sql files", ".sql" +- "sql files", ".sql" - "simple", "basic", "straightforward" - "aggregations", "joins", "transformations" - "materialized view", "CREATE OR REFRESH" @@ -258,8 +270,9 @@ When a user requests a new Lakeflow pipeline, Claude should detect the appropria - User mentions only data transformations without complex logic - Request focuses on filtering, joining, aggregating data - No mention of custom functions or external integrations +- **No explicit mention of "Python"** -**Default Behavior**: Prefer SQL when ambiguous (covers 90% of use cases) +**Default Behavior**: Prefer SQL only when ambiguous AND no Python indicators present ### Python Indicators @@ -280,11 +293,10 @@ When a user requests a new Lakeflow pipeline, Claude should detect the appropria ### Ambiguous Cases (Ask User) -**Indicators:** -- Both SQL and Python keywords mentioned -- "mixed pipeline", "some Python, some SQL" -- "complex pipeline" without specifics -- Unclear requirements or vague description +**Only ask when ALL conditions are met:** +- User did NOT explicitly mention "Python" or "SQL" +- Mixed signals present (some SQL keywords, some Python keywords) +- OR no clear indicators either way **Response:** ``` @@ -302,7 +314,7 @@ Which would you prefer? For bronze/silver/gold organization, Asset Bundles support two approaches. Both work with the `transformations/**` glob pattern in pipeline configuration. -### Option 1: Flat Structure with Naming (Template Default) +### Option 1: Flat Structure with Naming (Template Default, SQL Example) ``` transformations/ @@ -323,7 +335,7 @@ transformations/ - Simple file listing and discovery - Clear naming provides logical organization -### Option 2: Subdirectories by Layer +### Option 2: Subdirectories by Layer, SQL Example ``` transformations/ @@ -429,73 +441,6 @@ my_pipeline/ └── summary.sql ``` -**Migration Steps:** - -1. **Initialize new bundle project** - ```bash - databricks pipelines init --output-dir my_pipeline_bundle - cd my_pipeline_bundle/src/my_pipeline_bundle_etl/transformations/ - ``` - -2. **Copy files using either approach** - - **Option A: Flat structure with naming** - ```bash - # Remove sample files - rm sample_*.sql - - # Copy and rename with medallion prefix - cp ../../../../my_pipeline/bronze/orders.sql bronze_orders.sql - cp ../../../../my_pipeline/bronze/events.sql bronze_events.sql - cp ../../../../my_pipeline/silver/cleaned.sql silver_cleaned.sql - cp ../../../../my_pipeline/silver/joined.sql silver_joined.sql - cp ../../../../my_pipeline/gold/summary.sql gold_summary.sql - ``` - - **Option B: Keep subdirectories** - ```bash - # Remove sample files - rm sample_*.sql - - # Copy entire directory structure - cp -r ../../../../my_pipeline/bronze . - cp -r ../../../../my_pipeline/silver . - cp -r ../../../../my_pipeline/gold . - ``` - -3. **Update file references (if needed)** - - If files reference each other by path, update to use table names - - Example: Change `FROM ../bronze/orders` to `FROM LIVE.bronze_orders` - - Table names are derived from filenames or view/table definitions, not folder structure - -4. **Deploy bundle** - ```bash - cd ../../.. # Back to bundle root - databricks bundle deploy - databricks bundle run my_pipeline_bundle_etl - ``` - -**Benefits:** -- Multi-environment support (dev/staging/prod) -- Version control for configuration -- CI/CD integration -- Professional project structure - -### Option 2: Keep Manual Structure (Legacy) - -Continue using the manual workflow if: -- Quick prototyping without multi-environment needs -- Existing workflow is working well -- Team prefers manual control - -**Legacy Workflow:** -1. Write files in bronze/silver/gold folders -2. Upload with `upload_folder` MCP tool -3. Create pipeline with `create_or_update_pipeline` MCP tool -4. Update files and re-upload - -See [SKILL.md](SKILL.md) "Alternative: Manual Workflow" section for details. - --- ## Python Project: Dependency Management diff --git a/.claude/skills/databricks-spark-declarative-pipelines/9-auto_cdc.md b/.claude/skills/databricks-spark-declarative-pipelines/9-auto_cdc.md new file mode 100644 index 00000000..b8a5b59b --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/9-auto_cdc.md @@ -0,0 +1,353 @@ +# AUTO CDC Patterns for Change Data Capture + +**Keywords**: Slow Changing Dimension, SCD, SCD Type 1, SCD Type 2, AUTO CDC, change data capture, dp.create_auto_cdc_flow, deduplication + +--- + +## Overview + +AUTO CDC automatically handles Change Data Capture (CDC) to track changes in your data using Slow Changing Dimensions (SCD). It provides automatic deduplication, change tracking, and handles late-arriving data correctly. + +**Where to apply AUTO CDC:** +- **Silver layer**: When business users need deduplicated or historical data for analytics/ML +- **Gold layer**: When implementing dimensional modeling (star schema) with dim/fact tables +- **Choice depends on**: Downstream consumption patterns and query requirements + +--- + +## SCD Type 1 vs Type 2 + +### SCD Type 1 (In-place updates) +- **Overwrites** old values with new values +- **No history preserved** - only current state maintained +- **Use for**: Dimension attributes that don't need history + - Correcting data errors (typos) + - Updating attributes where history doesn't matter + - Maintaining single current record per key +- **Syntax**: `stored_as_scd_type="1"` (string) + +### SCD Type 2 (History tracking) +- **Creates new row** for each change +- **Preserves full history** with `__START_AT` and `__END_AT` timestamps +- **Use for**: Tracking changes over time + - Customer address changes + - Product price history + - Employee role changes + - Any dimension requiring temporal analysis +- **Syntax**: `stored_as_scd_type=2` (integer) + +--- + +## Pattern: Cleaning + AUTO CDC + +### Step 1: Clean and Validate Data + +Create a cleaned streaming table with proper typing and quality checks: + +```python +# Cleaned data preparation (can be silver or intermediate layer) +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +schema = spark.conf.get("schema") + +@dp.table( + name=f"{schema}.users_clean", + comment="Cleaned and validated user data with proper typing and quality checks", + cluster_by=["user_id"] +) +def users_clean(): + """ + Prepare clean data with: + - Proper timestamp typing + - Data quality validations + - Remove records with invalid email or null user_id + """ + return ( + spark.readStream.table("bronze_users") + .filter(F.col("user_id").isNotNull()) + .filter(F.col("email").isNotNull()) + .filter(F.col("email").rlike(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")) + .withColumn("created_timestamp", F.to_timestamp("created_timestamp")) + .withColumn("updated_timestamp", F.to_timestamp("updated_timestamp")) + .drop("_rescued_data") + .select( + "user_id", + "email", + "name", + "subscription_tier", + "country", + "created_timestamp", + "updated_timestamp", + "_ingested_at", + "_source_file" + ) + ) +``` + +### Step 2: Apply AUTO CDC (SCD Type 2) + +Create a history-tracked dimension table with full change history: + +```python +# AUTO CDC with SCD Type 2 (history tracking) +from pyspark import pipelines as dp + +target_schema = spark.conf.get("target_schema") +source_schema = spark.conf.get("source_schema") + +# Create the target table for AUTO CDC +dp.create_streaming_table(f"{target_schema}.dim_users") + +# Apply AUTO CDC (SCD Type 2) +dp.create_auto_cdc_flow( + target=f"{target_schema}.dim_users", + source=f"{source_schema}.users_clean", + keys=["user_id"], + sequence_by="updated_timestamp", + stored_as_scd_type=2 # Integer for Type 2 +) +``` + +**Resulting table will include**: +- All original columns from source +- `__START_AT` - When this version became effective +- `__END_AT` - When this version expired (NULL for current) + +### Step 3: Apply AUTO CDC (SCD Type 1) + +Create a deduplicated table with in-place updates (no history): + +```python +# AUTO CDC with SCD Type 1 (in-place updates) +from pyspark import pipelines as dp + +target_schema = spark.conf.get("target_schema") +source_schema = spark.conf.get("source_schema") + +# Create the target table for AUTO CDC +dp.create_streaming_table(f"{target_schema}.orders_current") + +# Apply AUTO CDC (SCD Type 1) +dp.create_auto_cdc_flow( + target=f"{target_schema}.orders_current", + source=f"{source_schema}.orders_clean", + keys=["order_id"], + sequence_by="updated_timestamp", + stored_as_scd_type="1" # String for Type 1 +) +``` + +--- + +## Key Benefits + +- **Automatic deduplication** based on keys - no manual MERGE logic +- **Automatic change tracking** with temporal metadata (`__START_AT`, `__END_AT`) +- **Handles late-arriving data** correctly using `sequence_by` timestamp +- **Simplified pipeline code** - no complex merge/upsert logic required +- **Built-in idempotency** - safe to reprocess data + +--- + +## Common Patterns + +### Pattern 1: Gold Dimensional Model + +Use AUTO CDC in Gold layer for star schema dimensions: + +```python +# Silver: Cleaned streaming tables +@dp.table(name="silver.customers_clean") +def customers_clean(): + return spark.readStream.table("bronze.customers").filter(...) + +# Gold: SCD Type 2 dimension +dp.create_streaming_table("gold.dim_customers") +dp.create_auto_cdc_flow( + target="gold.dim_customers", + source="silver.customers_clean", + keys=["customer_id"], + sequence_by="updated_at", + stored_as_scd_type=2 +) + +# Gold: Fact table (no AUTO CDC) +@dp.table(name="gold.fact_orders") +def fact_orders(): + return spark.read.table("silver.orders_clean") +``` + +### Pattern 2: Silver Deduplication for Joins + +Use AUTO CDC in Silver when joining multiple tables: + +```python +# Silver: AUTO CDC for deduplication +dp.create_streaming_table("silver.products_dedupe") +dp.create_auto_cdc_flow( + target="silver.products_dedupe", + source="bronze.products", + keys=["product_id"], + sequence_by="modified_at", + stored_as_scd_type="1" # Type 1: just dedupe, no history +) + +# Silver: Join with deduplicated data +@dp.table(name="silver.orders_enriched") +def orders_enriched(): + orders = spark.readStream.table("bronze.orders") + products = spark.read.table("silver.products_dedupe") + return orders.join(products, "product_id") +``` + +### Pattern 3: Mixed SCD Types + +Different tables use different SCD types based on requirements: + +```python +# SCD Type 2: Need history +dp.create_auto_cdc_flow( + target="gold.dim_customers", + source="silver.customers", + keys=["customer_id"], + sequence_by="updated_at", + stored_as_scd_type=2 # Track address changes over time +) + +# SCD Type 1: Corrections only +dp.create_auto_cdc_flow( + target="gold.dim_products", + source="silver.products", + keys=["product_id"], + sequence_by="modified_at", + stored_as_scd_type="1" # Current product info only +) +``` + +--- + +## Selective History Tracking + +Track history only for specific columns (SCD Type 2): + +```python +dp.create_auto_cdc_flow( + target="gold.dim_products", + source="silver.products_clean", + keys=["product_id"], + sequence_by="modified_at", + stored_as_scd_type=2, + track_history_column_list=["price", "cost"] # Only track these columns +) +``` + +When `price` or `cost` changes, a new version is created. Other column changes update the current record without creating new versions. + +--- + +## Using Temporary Views with AUTO CDC + +**`@dp.temporary_view()`** creates in-pipeline temporary views that exist only during pipeline execution. These are useful for intermediate transformations before AUTO CDC. + +**Key Constraints:** +- Cannot specify `catalog` or `schema` (temporary views are pipeline-scoped only) +- Cannot use `cluster_by` (not persisted) +- Only exists during pipeline execution + +**Use Cases:** +- Complex transformations before AUTO CDC +- Intermediate logic that's referenced multiple times +- Avoiding redundant transformations + +**Example: Preparation before AUTO CDC** + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +# Step 1: Temporary view for complex business logic +@dp.temporary_view() +def orders_with_calculated_fields(): + """ + Temporary view for complex calculations. + No catalog/schema needed - exists only in pipeline. + """ + return ( + spark.readStream.table("bronze.orders") + .withColumn("order_total", F.col("quantity") * F.col("unit_price")) + .withColumn("discount_amount", F.col("order_total") * F.col("discount_rate")) + .withColumn("final_amount", F.col("order_total") - F.col("discount_amount")) + .withColumn("order_category", + F.when(F.col("final_amount") > 1000, "large") + .when(F.col("final_amount") > 100, "medium") + .otherwise("small") + ) + .filter(F.col("order_id").isNotNull()) + .filter(F.col("final_amount") > 0) + .filter(F.col("order_date").isNotNull()) + ) + +# Step 2: Apply AUTO CDC using the temporary view as source +target_schema = spark.conf.get("target_schema") + +dp.create_streaming_table(f"{target_schema}.orders_current") +dp.create_auto_cdc_flow( + target=f"{target_schema}.orders_current", + source="orders_with_calculated_fields", # Reference temporary view by name + keys=["order_id"], + sequence_by="order_date", + stored_as_scd_type="1" +) +``` + +**Benefits:** +- Avoids creating unnecessary persisted tables +- Reduces storage costs (nothing written to disk) +- Simplifies complex multi-step transformations +- Enables code reuse across multiple tables in same pipeline + +--- + +## Related Documentation + +- **[3-scd-query-patterns.md](3-scd-query-patterns.md)** - Querying SCD Type 2 history tables, point-in-time analysis, temporal joins +- **[1-ingestion-patterns.md](1-ingestion-patterns.md)** - CDC data sources (Kafka, Event Hubs, Kinesis) +- **[2-streaming-patterns.md](2-streaming-patterns.md)** - Deduplication patterns without AUTO CDC + +--- + +## Best Practices + +1. **Choose the right SCD type**: + - Type 2 when you need to query historical states + - Type 1 when you only need current state or deduplication + +2. **Use meaningful sequence_by column**: + - Should reflect true chronological order of changes + - Typically `updated_timestamp`, `modified_at`, or `event_timestamp` + +3. **Clean data before AUTO CDC**: + - Apply type casting, validation, and filtering first + - AUTO CDC works best with clean, well-typed data + +4. **Consider query patterns**: + - If analysts query history → Use Type 2 + - If analysts only need current → Use Type 1 + - If joining frequently → Consider Silver deduplication + +5. **Use selective tracking for large tables**: + - Track history only for columns that change meaningfully + - Reduces storage and improves query performance + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Duplicates still appearing** | Check `keys` include all business key columns; verify `sequence_by` has proper ordering | +| **Missing `__START_AT`/`__END_AT` columns** | These only appear in SCD Type 2 (integer), not Type 1 (string) | +| **Late data not handled** | Ensure `sequence_by` column is set and reflects true event time | +| **Type syntax error** | Type 2 uses integer `2`, Type 1 uses string `"1"` | +| **Performance issues** | Use `track_history_column_list` to limit which columns trigger new versions | diff --git a/.claude/skills/databricks-spark-declarative-pipelines/SKILL.md b/.claude/skills/databricks-spark-declarative-pipelines/SKILL.md new file mode 100644 index 00000000..144041e2 --- /dev/null +++ b/.claude/skills/databricks-spark-declarative-pipelines/SKILL.md @@ -0,0 +1,577 @@ +--- +name: databricks-spark-declarative-pipelines +description: "Creates, configures, and updates Databricks Lakeflow Spark Declarative Pipelines (SDP/LDP) using serverless compute. Handles streaming tables, materialized views, CDC, SCD Type 2, and Auto Loader ingestion patterns. Use when building data pipelines, working with Delta Live Tables, ingesting streaming data, implementing change data capture, or when the user mentions SDP, LDP, DLT, Lakeflow pipelines, streaming tables, or bronze/silver/gold medallion architectures." +--- + +# Lakeflow Spark Declarative Pipelines (SDP) + +IMPORTANT: If this is a new pipeline (one does not already exist), see Quick Start. Be sure to use whatever language user has specified only (Python or SQL). Be sure to use Databricks Asset Bundles for new projects. + +--- + +## Critical Rules (always follow) +- **MUST** confirm language as Python or SQL. Stick with that language unless told otherwise. +- **MUST** if not modifying an existing pipeline, use [Quick Start](#quick-start) below. +- **MUST** create serverless pipelines by default. ** Only use classic clusters if user explicitly requires R language, Spark RDD APIs, or JAR libraries. + + +## Required Steps + +Copy this checklist and verify each item: +``` +- [ ] Language selected: Python or SQL +- [ ] Compute type decided: serverless or classic compute +- [ ] Decide on multiple catalogs or schemas vs. all in one default schema +- [ ] Consider what should be parameterized at the pipeline level to make deployment easy. +- [ ] Consider [Multi-Schema Patterns](#multi-schema-patterns) below, ask if unclear on best choices. +- [ ] Consider [Modern Defaults](#modern-defaults) below, ask if unclear on best choices. + + +## Quick Start: Initialize New Pipeline Project + +**RECOMMENDED**: Use `databricks pipelines init` to create production-ready Asset Bundle projects with multi-environment support. + +### When to Use Bundle Initialization + +Use bundle initialization for **New pipeline projects** for a professional structure from the start + +Use manual workflow for: +- Quick prototyping without multi-environment needs +- Existing manual projects you want to continue +- Learning/experimentation + +### Step 1: Initialize Project + +I will automatically run this command when you request a new pipeline: + +```bash +databricks pipelines init +``` + +**Interactive Prompts:** +- **Project name**: e.g., `customer_orders_pipeline` +- **Initial catalog**: Unity Catalog name (e.g., `main`, `prod_catalog`) +- **Personal schema per user?**: `yes` for dev (each user gets their own schema), `no` for prod +- **Language**: SQL or Python (auto-detected from your request - see language detection below) + +**Generated Structure:** +``` +my_pipeline/ +├── databricks.yml # Multi-environment config (dev/prod) +├── resources/ +│ └── *_etl.pipeline.yml # Pipeline resource definition +└── src/ + └── *_etl/ + ├── explorations/ # Exploratory code in .ipynb + └── transformations/ # Your .sql or .py files here +``` + +### Step 2: Customize Transformations + +Replace the example code created by the init process with custom transformation files in `src/transformations/` based on provided requirements, using best practice guidance from this skill. + +**For Python pipelines using cloudFiles**: Ask the user where to store Auto Loader schema metadata. Recommend: +``` +/Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas +``` + +### Step 3: Deploy and Run + +```bash +# Deploy to workspace (dev by default) +databricks bundle deploy + +# Run pipeline +databricks bundle run my_pipeline_etl + +# Deploy to production +databricks bundle deploy --target prod +``` + + +## Quick Reference + +| Concept | Details | +|---------|---------| +| **Names** | SDP = Spark Declarative Pipelines = LDP = Lakeflow Declarative Pipelines = Lakeflow Pipelines (all interchangeable) | +| **Python Import** | `from pyspark import pipelines as dp` | +| **Primary Decorators** | `@dp.table()`, `@dp.materialized_view()`, `@dp.temporary_view()` | +| **Temporary Views** | `@dp.temporary_view()` creates in-pipeline temporary views (no catalog/schema, no cluster_by). Useful for intermediate logic before AUTO CDC or when a view needs multiple references without persistence. | +| **Replaces** | Delta Live Tables (DLT) with `import dlt` | +| **Based On** | Apache Spark 4.1+ (Databricks' modern data pipeline framework) | +| **Docs** | https://docs.databricks.com/aws/en/ldp/developer/python-dev | + +--- + +## Detailed guides + +**Ingestion patterns**: Use [1-ingestion-patterns.md](1-ingestion-patterns.md) when planning how to get new data into your Lakeflow pipeline —- covers file formats, batch/streaming options, and tips for incremental and full loads. (Keywords: Auto Loader, Kafka, Event Hub, Kinesis, file formats) + +**Streaming pipeline patterns**: See [2-streaming-patterns.md](2-streaming-patterns.md) for designing pipelines with streaming data sources, change data detection, triggers, and windowing. (Keywords: deduplication, windowing, stateful operations, joins) + +**SCD query patterns**: See [3-scd-query-patterns.md](3-scd-query-patterns.md) for querying Slowly Changing Dimensions Type 2 history tables, including current state queries, point-in-time analysis, temporal joins, and change tracking. (Keywords: SCD Type 2 history tables, temporal joins, querying historical data) + +**Performance tuning**: Use [4-performance-tuning.md](4-performance-tuning.md) for optimizing pipelines with Liquid Clustering, state management, and best practices for high-performance streaming workloads. (Keywords: Liquid Clustering, optimization, state management) + +**Python API reference**: See [5-python-api.md](5-python-api.md) for the modern `pyspark.pipelines` (dp) API reference and migration from legacy `dlt` API patterns. (Keywords: dp API, dlt API comparison) + +**DLT migration**: Use [6-dlt-migration.md](6-dlt-migration.md) when migrating existing Delta Live Tables (DLT) pipelines to Spark Declarative Pipelines (SDP). (Keywords: migrating DLT pipelines to SDP) + +**Advanced configuration**: See [7-advanced-configuration.md](7-advanced-configuration.md) for advanced pipeline settings including development mode, continuous execution, notifications, Python dependencies, and custom cluster configurations. (Keywords: extra_settings parameter reference, examples) + +**Project initialization**: Use [8-project-initialization.md](8-project-initialization.md) for setting up new pipeline projects with `databricks pipelines init`, Asset Bundles, multi-environment deployments, and language detection logic. (Keywords: databricks pipelines init, Asset Bundles, language detection, migration guides) + +**AUTO CDC patterns**: Use [9-auto_cdc.md](9-auto_cdc.md) for implementing Change Data Capture with AUTO CDC, including Slow Changing Dimensions (SCD Type 1 and Type 2) for tracking changes and deduplication. (Keywords: AUTO CDC, Slow Changing Dimension, SCD, SCD Type 1, SCD Type 2, change data capture, deduplication) + +--- + +## Workflow + +1. Determine the task type: + + **Setting up new project?** → Read [8-project-initialization.md](8-project-initialization.md) first + **Creating new pipeline?** → Read [1-ingestion-patterns.md](1-ingestion-patterns.md) + **Creating stream table?** → Read [2-streaming-patterns.md](2-streaming-patterns.md) + **Querying SCD history tables?** → Read [3-scd-query-patterns.md](3-scd-query-patterns.md) + **Implementing AUTO CDC or SCD?** → Read [9-auto_cdc.md](9-auto_cdc.md) + **Performance issues?** → Read [4-performance-tuning.md](4-performance-tuning.md) + **Using Python API?** → Read [5-python-api.md](5-python-api.md) + **Migrating from DLT?** → Read [6-dlt-migration.md](6-dlt-migration.md) + **Advanced configuration?** → Read [7-advanced-configuration.md](7-advanced-configuration.md) + **Validating?** → Read [validation-checklist.md](validation-checklist.md) + +2. Follow the instructions in the relevant guide + +3. Repeat for next task type +--- + +## Official Documentation + +- **[Lakeflow Spark Declarative Pipelines Overview](https://docs.databricks.com/aws/en/ldp/)** - Main documentation hub +- **[SQL Language Reference](https://docs.databricks.com/aws/en/ldp/developer/sql-dev)** - SQL syntax for streaming tables and materialized views +- **[Python Language Reference](https://docs.databricks.com/aws/en/ldp/developer/python-ref)** - `pyspark.pipelines` API +- **[Loading Data](https://docs.databricks.com/aws/en/ldp/load)** - Auto Loader, Kafka, Kinesis ingestion +- **[Change Data Capture (CDC)](https://docs.databricks.com/aws/en/ldp/cdc)** - AUTO CDC, SCD Type 1/2 + + +### Medallion Architecture Pattern + **Bronze Layer (Raw)** + - Raw data ingested from sources in original format + - Minimal transformations (append-only, add metadata like `_ingested_at`, `_source_file`) + - Single source of truth preserving data lineage + + **Silver Layer (Validated)** + - Cleaned and validated data. + - Might deduplicate here with auto_cdc, but often wait until the final step for auto_cdc if possible. + - Business logic applied (type casting, quality checks, filtering invalid records) + - Enterprise view of key business entities + - Enables self-service analytics and ML + + **Gold Layer (Business-Ready)** + - Aggregated, denormalized, project-specific tables + - Optimized for consumption (reporting, dashboards, BI tools) + - Fewer joins, read-optimized data models + - Kimball star schema tables - dim_, fact_ + - Deduplication often happens here via Slow Changing Dimensions (SCD), using auto_cdc. Sometimes that will happen upstream in silver instead, such as when joining multiple tables or business users plan to query the table from silver. + + **Typical Flow (Can vary)** + Bronze: read_files() or spark.readStream.format("cloudFiles") → streaming table + Silver: read bronze → filter/clean/validate → streaming table + Gold: read silver → aggregate/denormalize → auto_cdc or materialized view + + Sources: + - https://www.databricks.com/glossary/medallion-architecture + - https://docs.databricks.com/aws/en/lakehouse/medallion + - https://www.databricks.com/blog/2022/06/24/data-warehousing-modeling-techniques-and-their-implementation-on-the-databricks-lakehouse-platform.html + +**For medallion architecture** (bronze/silver/gold), two approaches work: +- **Flat with naming** (template default): `bronze_*.sql`, `silver_*.sql`, `gold_*.sql` +- **Subdirectories**: `bronze/orders.sql`, `silver/cleaned.sql`, `gold/summary.sql` + +Both work with the `transformations/**` glob pattern. Choose based on preference. + +See **[8-project-initialization.md](8-project-initialization.md)** for complete details on bundle initialization, migration, and troubleshooting. + +--- +## General SDP development guidance +### Step 1: Write Pipeline Files Locally + +Create `.sql` or `.py` files in a local folder: + +``` +my_pipeline/ +├── bronze/ +│ ├── ingest_orders.sql # SQL (default for most cases) +│ └── ingest_events.py # Python (for complex logic) +├── silver/ +│ └── clean_orders.sql +└── gold/ + └── daily_summary.sql +``` + +**SQL Example** (`bronze/ingest_orders.sql`): +```sql +CREATE OR REFRESH STREAMING TABLE bronze_orders +CLUSTER BY (order_date) +AS +SELECT + *, + current_timestamp() AS _ingested_at, + _metadata.file_path AS _source_file +FROM read_files( + '/Volumes/catalog/schema/raw/orders/', + format => 'json', + schemaHints => 'order_id STRING, customer_id STRING, amount DECIMAL(10,2), order_date DATE' +); +``` + +**Python Example** (`bronze/ingest_events.py`): +```python +from pyspark import pipelines as dp +from pyspark.sql.functions import col, current_timestamp + +# Get schema location from pipeline configuration +schema_location_base = spark.conf.get("schema_location_base") + +@dp.table(name="bronze_events", cluster_by=["event_date"]) +def bronze_events(): + return ( + spark.readStream.format("cloudFiles") + .option("cloudFiles.format", "json") + .option("cloudFiles.schemaLocation", f"{schema_location_base}/bronze_events") + .load("/Volumes/catalog/schema/raw/events/") + .withColumn("_ingested_at", current_timestamp()) + .withColumn("_source_file", col("_metadata.file_path")) + ) +``` + +**IMPORTANT for Python Pipelines**: When using `spark.readStream.format("cloudFiles")` for cloud storage ingestion, with schema inference (no schema specified), you **must specify a schema location**. + +**Always ask the user** where to store Auto Loader schema metadata. Recommend: +``` +/Volumes/{catalog}/{schema}/{pipeline_name}_metadata/schemas +``` + +Example: `/Volumes/my_catalog/pipeline_metadata/orders_pipeline_metadata/schemas` + +**Never use the source data volume** - this causes permission conflicts. The schema location should be configured in the pipeline settings and accessed via `spark.conf.get("schema_location_base")`. + +**Language Selection:** + +**CRITICAL RULE**: If the user explicitly mentions "Python" in their request (e.g., "Python Spark Declarative Pipeline", "Python SDP", "use Python"), **ALWAYS use Python without asking**. The same applies to SQL - if they say "SQL pipeline", use SQL. + +- **Explicit language request**: User says "Python" → Use Python. User says "SQL" → Use SQL. **Do not ask for clarification.** +- **Auto-detection** (only when no explicit language mentioned): + - **SQL indicators**: "sql files", "simple transformations", "aggregations", "materialized view", "CREATE OR REFRESH" + - **Python indicators**: ".py files", "UDF", "complex logic", "ML inference", "external API", "@dp.table", "pandas", "decorator" +- **Prompt for clarification** only when language intent is truly ambiguous (no explicit mention, mixed signals) +- **Default to SQL** only when ambiguous AND no Python indicators present + +See **[8-project-initialization.md](8-project-initialization.md)** for detailed language detection logic. + + +## Option 1: Pipelines with DABs: +Use asset bundles and pipeline CLI. +See [Quick Start](#quick-start) and **[8-project-initialization.md](8-project-initialization.md)** for complete details. + +## Option 2: Manual Workflow (Advanced) + +For rapid prototyping, experimentation, or when you prefer direct control without Asset Bundles, use the manual workflow with MCP tools. + +Use MCP tools to create, run, and iterate on **serverless SDP pipelines**. The **primary tool is `create_or_update_pipeline`** which handles the entire lifecycle. + +**IMPORTANT: Always create serverless pipelines (default).** Only use classic clusters if user explicitly ask for classic, pro, advances compute or requires R language, Spark RDD APIs, or JAR libraries. + +See **[10-mcp-approach.md](10-mcp-approach.md)** for detailed guide. + + +## Best Practices (2026) + +### Project Structure +- **Default to `databricks pipelines init`** for new projects (creates Asset Bundle) +- **Use Asset Bundles** for multi-environment deployments (dev/staging/prod) +- **Manual structure only** for quick prototypes or legacy migration +- **Medallion architecture**: Two approaches work with Asset Bundles: + - **Flat structure** (template default): `bronze_*.sql`, `silver_*.sql`, `gold_*.sql` in `transformations/` + - **Subdirectories**: `transformations/bronze/`, `transformations/silver/`, `transformations/gold/` + - Both work with the `transformations/**` glob pattern - choose based on team preference +- See **[8-project-initialization.md](8-project-initialization.md)** for project setup details + +### Minimal pipeline config pointers +- Define parameters in your pipeline’s configuration and access them in code with spark.conf.get("key"). +- In Databricks Asset Bundles, set these under resources.pipelines..configuration; validate with databricks bundle validate. + +### Modern Defaults +- **CLUSTER BY** (Liquid Clustering), not PARTITION BY - see [4-performance-tuning.md](4-performance-tuning.md) +- **Raw `.sql`/`.py` files**, not notebooks +- **Serverless compute ONLY** - Do not use classic clusters unless explicitly required +- **Unity Catalog** (required for serverless) +- **read_files()** when using SQL for cloud storage ingestion - see [1-ingestion-patterns.md](1-ingestion-patterns.md) + +### Multi-Schema Patterns + +**Default: Single target schema per pipeline.** Each pipeline has one target `catalog` and `schema` where all tables are written. + + +#### Option 1: Single Pipeline, Single Schema with Prefixes (Recommended) + +Use one schema with table name prefixes to distinguish layers: + +```python +# All tables write to: catalog.schema.bronze_*, silver_*, gold_* +@dp.table(name="bronze_orders") # → catalog.schema.bronze_orders +@dp.table(name="silver_orders") # → catalog.schema.silver_orders +@dp.table(name="gold_summary") # → catalog.schema.gold_summary +``` + +**Advantages:** +- Simpler configuration (one pipeline) +- All tables in one schema for easy discovery + +#### Option 2: +Use varaiables to specific separate catalog and/or schema for different steps. + +Below are Python SDP examples that source variables from pipeline configs via spark.conf.get, and use the default catalog/schema for bronze. + +##### Same catalog, separate schemas; bronze uses pipeline defaults +- Set your pipeline’s default catalog and default schema to the bronze layer (for example, catalog=my_catalog, schema=bronze). When you omit catalog/schema in code, reads/writes go to these defaults. +- Use pipeline parameters for the other schemas and any source schema/path, retrieved in code with spark.conf.get(...). + +```python +from pyspark import pipelines as dp +from pyspark.sql.functions import col + +# Pull variables from pipeline configuration parameters +silver_schema = spark.conf.get("silver_schema") # e.g., "silver" +gold_schema = spark.conf.get("gold_schema") # e.g., "gold" +landing_schema = spark.conf.get("landing_schema") # e.g., "landing" + +# Bronze → uses default catalog/schema (set to bronze in pipeline settings) +@dp.table(name="orders_bronze") +def orders_bronze(): + # Read from another schema in the same default catalog + return spark.readStream.table(f"{landing_schema}.orders_raw") + +# Silver → same catalog, schema from parameter +@dp.table(name=f"{silver_schema}.orders_clean") +def orders_clean(): + return (spark.read.table("orders_bronze") # unqualified = default catalog/schema + .filter(col("order_id").isNotNull())) + +# Gold → same catalog, schema from parameter +@dp.materialized_view(name=f"{gold_schema}.orders_by_date") +def orders_by_date(): + return (spark.read.table(f"{silver_schema}.orders_clean") + .groupBy("order_date") + .count().withColumnRenamed("count", "order_count")) +``` +- Using unqualified names for bronze ensures it lands in the pipeline’s default catalog/schema; silver/gold are explicitly schema-qualified within the same catalog. + +--- + +##### Custom catalog/schema per layer; bronze still uses pipeline defaults +- Keep bronze in the pipeline defaults (default catalog/schema set to your bronze layer). For silver/gold, use fully-qualified names with catalog and schema variables from pipeline configuration. + +```python +from pyspark import pipelines as dp +from pyspark.sql.functions import col + +# Pull variables from pipeline configuration parameters +silver_catalog = spark.conf.get("silver_catalog") # e.g., "my_catalog" +silver_schema = spark.conf.get("silver_schema") # e.g., "silver" +gold_catalog = spark.conf.get("gold_catalog") # e.g., "my_catalog" +gold_schema = spark.conf.get("gold_schema") # e.g., "gold" +landing_catalog = spark.conf.get("landing_catalog") # optional, if source is in another catalog +landing_schema = spark.conf.get("landing_schema") + +# Bronze → uses default catalog/schema (set to bronze) +@dp.table(name="orders_bronze") +def orders_bronze(): + # If source is in a specified catalog/schema: + return spark.readStream.table(f"{landing_catalog}.{landing_schema}.orders_raw") + +# Silver → custom catalog + schema via parameters +@dp.table(name=f"{silver_catalog}.{silver_schema}.orders_clean") +def orders_clean(): + # Read bronze by its unqualified name (defaults), or fully qualify if preferred + return (spark.read.table("orders_bronze") + .filter(col("order_id").isNotNull())) + +# Gold → custom catalog + schema via parameters +@dp.materialized_view(name=f"{gold_catalog}.{gold_schema}.orders_by_date}") +def orders_by_date(): + return (spark.read.table(f"{silver_catalog}.{silver_schema}.orders_clean") + .groupBy("order_date") + .count().withColumnRenamed("count", "order_count")) +``` +- Multipart names in the decorator’s name argument let you publish to explicit catalog.schema targets within one pipeline. +- Unqualified reads/writes use the pipeline defaults; use fully-qualified names when crossing catalogs or when you need explicit namespace control. + +--- + + +**Note:** The `@dp.table()` decorator does not currently support separate for `schema=` or `catalog=` parameters. The table parameter is a string that contains the catalog.schema.table_name, or it can leave off catalog and or schema to use the pipeilnes configured default target schema. + +### Reading Tables in Python + +**Modern SDP Best Practice:** +- Use `spark.read.table()` for batch reads +- Use `spark.readStream.table()` for streaming reads +- Don't use `dp.read()` or `dp.read_stream()` (old syntax, no longer documented) +- Don't use `dlt.read()` or `dlt.read_stream()` (legacy DLT API) + +**Key Point:** SDP automatically tracks table dependencies from standard Spark DataFrame operations. No special read APIs are needed. + +#### Three-Tier Identifier Resolution + +SDP supports three levels of table name qualification: + +| Level | Syntax | When to Use | +|-------|--------|-------------| +| **Unqualified** | `spark.read.table("my_table")` | Reading tables within the same pipeline's target catalog/schema (recommended) | +| **Partially-qualified** | `spark.read.table("other_schema.my_table")` | Reading from different schema in same catalog | +| **Fully-qualified** | `spark.read.table("other_catalog.other_schema.my_table")` | Reading from external catalogs/schemas | + +#### Option 1: Unqualified Names (Recommended for Pipeline Tables) + +**Best practice for tables within the same pipeline.** SDP resolves unqualified names to the pipeline's configured target catalog and schema. This makes code portable across environments (dev/prod). + +```python +@dp.table(name="silver_clean") +def silver_clean(): + # Reads from pipeline's target catalog/schema (e.g., dev_catalog.dev_schema.bronze_raw) + return ( + spark.read.table("bronze_raw") + .filter(F.col("valid") == True) + ) + +@dp.table(name="silver_events") +def silver_events(): + # Streaming read from same pipeline's bronze_events table + return ( + spark.readStream.table("bronze_events") + .withColumn("processed_at", F.current_timestamp()) + ) +``` + +#### Option 2: Pipeline Parameters (For External Sources) + +**Use `spark.conf.get()` to parameterize external catalog/schema references.** Define parameters in pipeline configuration, then reference them at the module level. + +```python +from pyspark import pipelines as dp +from pyspark.sql import functions as F + +# Get parameterized values at module level (evaluated once at pipeline start) +source_catalog = spark.conf.get("source_catalog") +source_schema = spark.conf.get("source_schema", "sales") # with default + +@dp.table(name="transaction_summary") +def transaction_summary(): + return ( + spark.read.table(f"{source_catalog}.{source_schema}.transactions") + .groupBy("account_id") + .agg( + F.count("txn_id").alias("txn_count"), + F.sum("txn_amount").alias("account_revenue") + ) + ) +``` + +**Configure parameters in pipeline settings:** +- **Asset Bundles**: Add to `pipeline.yml` under `configuration:` +- **Manual/MCP**: Pass via `extra_settings.configuration` dict + +```yaml +# In resources/my_pipeline.pipeline.yml +configuration: + source_catalog: "shared_catalog" + source_schema: "sales" +``` + +#### Option 3: Fully-Qualified Names (For Fixed External References) + +Use when referencing specific external tables that don't change across environments: + +```python +@dp.table(name="enriched_orders") +def enriched_orders(): + # Pipeline-internal table (unqualified) + orders = spark.read.table("bronze_orders") + + # External reference table (fully-qualified) + products = spark.read.table("shared_catalog.reference.products") + + return orders.join(products, "product_id") +``` + +#### Choosing the Right Approach + +| Scenario | Recommended Approach | +|----------|---------------------| +| Reading tables created in same pipeline | **Unqualified names** - portable, uses target catalog/schema | +| Reading from external source that varies by environment | **Pipeline parameters** - configurable per deployment | +| Reading from shared/reference tables with fixed location | **Fully-qualified names** - explicit and clear | +| Mixed pipeline (some internal, some external) | **Combine approaches** - unqualified for internal, parameters for external | + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Empty output tables** | Use `get_table_details` to verify, check upstream sources | +| **Pipeline stuck INITIALIZING** | Normal for serverless, wait a few minutes | +| **"Column not found"** | Check `schemaHints` match actual data | +| **Streaming reads fail** | For file ingestion in a streaming table, you must use the `STREAM` keyword with `read_files`: `FROM STREAM read_files(...)`. For table streams use `FROM stream(table)`. See [read_files — Usage in streaming tables](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files#usage-in-streaming-tables). | +| **Timeout during run** | Increase `timeout`, or use `wait_for_completion=False` and poll with `get_update` | +| **MV doesn't refresh** | Enable row tracking on source tables | +| **SCD2: query column not found** | Lakeflow uses `__START_AT` and `__END_AT` (double underscore), not `START_AT`/`END_AT`. Use `WHERE __END_AT IS NULL` for current rows. See [3-scd-patterns.md](3-scd-patterns.md). | +| **AUTO CDC parse error at APPLY/SEQUENCE** | Put `APPLY AS DELETE WHEN` **before** `SEQUENCE BY`. Only list columns in `COLUMNS * EXCEPT (...)` that exist in the source (omit `_rescued_data` unless bronze uses rescue data). Omit `TRACK HISTORY ON *` if it causes "end of input" errors; default is equivalent. See [2-streaming-patterns.md](2-streaming-patterns.md). | +| **"Cannot create streaming table from batch query"** | In a streaming table query, use `FROM STREAM read_files(...)` so `read_files` leverages Auto Loader; `FROM read_files(...)` alone is batch. See [1-ingestion-patterns.md](1-ingestion-patterns.md) and [read_files — Usage in streaming tables](https://docs.databricks.com/aws/en/sql/language-manual/functions/read_files#usage-in-streaming-tables). | + +**For detailed errors**, the `result["message"]` from `create_or_update_pipeline` includes suggested next steps. Use `get_pipeline_events(pipeline_id=...)` for full stack traces. + +--- + +## Advanced Pipeline Configuration + +For advanced configuration options (development mode, continuous pipelines, custom clusters, notifications, Python dependencies, etc.), see **[7-advanced-configuration.md](7-advanced-configuration.md)**. + +--- + +## Platform Constraints + +### Serverless Pipeline Requirements (Default) +| Requirement | Details | +|-------------|---------| +| **Unity Catalog** | Required - serverless pipelines always use UC | +| **Workspace Region** | Must be in serverless-enabled region | +| **Serverless Terms** | Must accept serverless terms of use | +| **CDC Features** | Requires serverless (or Pro/Advanced with classic clusters) | + +### Serverless Limitations (When Classic Clusters Required) +| Limitation | Workaround | +|------------|-----------| +| **R language** | Not supported - use classic clusters if required | +| **Spark RDD APIs** | Not supported - use classic clusters if required | +| **JAR libraries** | Not supported - use classic clusters if required | +| **Maven coordinates** | Not supported - use classic clusters if required | +| **DBFS root access** | Limited - must use Unity Catalog external locations | +| **Global temp views** | Not supported | + +### General Constraints +| Constraint | Details | +|------------|---------| +| **Schema Evolution** | Streaming tables require full refresh for incompatible changes | +| **SQL Limitations** | PIVOT clause unsupported | +| **Sinks** | Python only, streaming only, append flows only | + +**Default to serverless** unless user explicitly requires R, RDD APIs, or JAR libraries. + +## Related Skills + +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - for orchestrating and scheduling pipeline runs +- **[databricks-asset-bundles](../databricks-asset-bundles/SKILL.md)** - for multi-environment deployment of pipeline projects +- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - for generating test data to feed into pipelines +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for catalog/schema/volume management and governance diff --git a/.claude/skills/databricks-spark-structured-streaming/SKILL.md b/.claude/skills/databricks-spark-structured-streaming/SKILL.md new file mode 100644 index 00000000..b1f59306 --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/SKILL.md @@ -0,0 +1,65 @@ +--- +name: databricks-spark-structured-streaming +description: Comprehensive guide to Spark Structured Streaming for production workloads. Use when building streaming pipelines, implementing real-time data processing, handling stateful operations, or optimizing streaming performance. +--- + +# Spark Structured Streaming + +Production-ready streaming pipelines with Spark Structured Streaming. This skill provides navigation to detailed patterns and best practices. + +## Quick Start + +```python +from pyspark.sql.functions import col, from_json + +# Basic Kafka to Delta streaming +df = (spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", "broker:9092") + .option("subscribe", "topic") + .load() + .select(from_json(col("value").cast("string"), schema).alias("data")) + .select("data.*") +) + +df.writeStream \ + .format("delta") \ + .outputMode("append") \ + .option("checkpointLocation", "/Volumes/catalog/checkpoints/stream") \ + .trigger(processingTime="30 seconds") \ + .start("/delta/target_table") +``` + +## Core Patterns + +| Pattern | Description | Reference | +|---------|-------------|-----------| +| **Kafka Streaming** | Kafka to Delta, Kafka to Kafka, Real-Time Mode | See [kafka-streaming.md](kafka-streaming.md) | +| **Stream Joins** | Stream-stream joins, stream-static joins | See [stream-stream-joins.md](stream-stream-joins.md), [stream-static-joins.md](stream-static-joins.md) | +| **Multi-Sink Writes** | Write to multiple tables, parallel merges | See [multi-sink-writes.md](multi-sink-writes.md) | +| **Merge Operations** | MERGE performance, parallel merges, optimizations | See [merge-operations.md](merge-operations.md) | + +## Configuration + +| Topic | Description | Reference | +|-------|-------------|-----------| +| **Checkpoints** | Checkpoint management and best practices | See [checkpoint-best-practices.md](checkpoint-best-practices.md) | +| **Stateful Operations** | Watermarks, state stores, RocksDB configuration | See [stateful-operations.md](stateful-operations.md) | +| **Trigger & Cost** | Trigger selection, cost optimization, RTM | See [trigger-and-cost-optimization.md](trigger-and-cost-optimization.md) | + +## Best Practices + +| Topic | Description | Reference | +|-------|-------------|-----------| +| **Production Checklist** | Comprehensive best practices | See [streaming-best-practices.md](streaming-best-practices.md) | + +## Production Checklist + +- [ ] Checkpoint location is persistent (UC volumes, not DBFS) +- [ ] Unique checkpoint per stream +- [ ] Fixed-size cluster (no autoscaling for streaming) +- [ ] Monitoring configured (input rate, lag, batch duration) +- [ ] Exactly-once verified (txnVersion/txnAppId) +- [ ] Watermark configured for stateful operations +- [ ] Left joins for stream-static (not inner) diff --git a/.claude/skills/databricks-spark-structured-streaming/checkpoint-best-practices.md b/.claude/skills/databricks-spark-structured-streaming/checkpoint-best-practices.md new file mode 100644 index 00000000..349cb9bf --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/checkpoint-best-practices.md @@ -0,0 +1,316 @@ +--- +name: checkpoint-best-practices +description: Configure and manage checkpoint locations for reliable Spark Structured Streaming. Use when setting up new streaming jobs, troubleshooting checkpoint issues, migrating checkpoints, or ensuring exactly-once semantics with proper checkpoint storage and organization. +--- + +# Checkpoint Best Practices + +Configure checkpoint locations for reliable streaming with exactly-once semantics. Checkpoints track progress and enable fault tolerance. + +## Quick Start + +```python +def get_checkpoint_location(table_name): + """Checkpoint tied to target table""" + return f"/Volumes/catalog/checkpoints/{table_name}" + +# Example: +# Table: prod.analytics.orders +# Checkpoint: /Volumes/prod/checkpoints/orders + +query = (df + .writeStream + .format("delta") + .option("checkpointLocation", get_checkpoint_location("orders")) + .start("/delta/orders") +) +``` + +## Checkpoint Storage + +### Use Persistent Storage + +```python +# DO: Use Unity Catalog volumes (S3/ADLS-backed) +checkpoint_path = "/Volumes/catalog/checkpoints/stream_name" + +# DON'T: Use DBFS (ephemeral, workspace-local) +checkpoint_path = "/dbfs/checkpoints/stream_name" # Avoid +``` + +### Target-Tied Organization + +```python +def get_checkpoint_location(table_name): + """Checkpoint should be tied to TARGET, not source""" + return f"/Volumes/catalog/checkpoints/{table_name}" + +# Why target-tied? +# - Checkpoint already contains source information +# - Systematic organization +# - Easy backup and restore +# - Clear ownership +``` + +### Unique Checkpoint Per Stream + +```python +# CORRECT: Each stream has its own checkpoint +stream1.writeStream \ + .option("checkpointLocation", "/checkpoints/stream1") \ + .start() + +stream2.writeStream \ + .option("checkpointLocation", "/checkpoints/stream2") \ + .start() + +# WRONG: Never share checkpoints between streams +# This causes data loss and corruption +``` + +## Checkpoint Structure + +### Folder Contents + +``` +checkpoint_location/ +├── metadata/ # Query ID +├── offsets/ # What to process (intent) +├── commits/ # What completed (confirmation) +├── sources/ # Source metadata +└── state/ # Stateful operations (if any) +``` + +### Stateless vs Stateful + +```python +# Stateless (read from Kafka, write to Delta) +# Checkpoint: metadata, offsets, commits, sources +# No state folder + +df = (spark.readStream + .format("kafka") + .option("subscribe", "topic") + .load()) + +# Stateful (with watermark and deduplication) +# Checkpoint: + state folder +df_stateful = (df + .withWatermark("timestamp", "10 minutes") + .dropDuplicates(["partition", "offset"]) +) +``` + +## Reading Checkpoint Contents + +### Read Offset Files + +```python +import json + +# Read offset file +offset_file = "/checkpoints/stream/offsets/223" +content = dbutils.fs.head(offset_file) +offset_data = json.loads(content) + +# Pretty print +print(json.dumps(offset_data, indent=2)) + +# Key fields: +# - batchWatermarkMs: Watermark timestamp +# - batchTimestampMs: When batch started +# - source[0].startOffset: Beginning of batch (inclusive) +# - source[0].endOffset: End of batch (exclusive) +# - source[0].latestOffset: Current position in source +``` + +### Read State Store + +```python +# Query state store directly +state_df = (spark + .read + .format("statestore") + .load("/checkpoints/stream/state") +) + +state_df.show() +# Shows: key, value, partitionId, expiration timestamp + +# Read state metadata +state_metadata = (spark + .read + .format("state-metadata") + .load("/checkpoints/stream") +) +state_metadata.show() +# Shows: operatorName, numPartitions, minBatchId, maxBatchId +``` + +## Recovery Scenarios + +### Lost Checkpoint + +```python +# Steps to recover: +# 1. Delete checkpoint folder +dbutils.fs.rm("/checkpoints/stream", recurse=True) + +# 2. Restart stream with startingOffsets=earliest +df.writeStream \ + .format("delta") \ + .option("checkpointLocation", "/checkpoints/stream") \ + .option("startingOffsets", "earliest") \ + .start() + +# 3. Stream reprocesses from beginning +# 4. Delta sink handles deduplication (if idempotent writes configured) +``` + +### Corrupted Checkpoint + +```python +# Same as lost checkpoint: +# 1. Delete checkpoint folder +# 2. Restart with startingOffsets=earliest +# 3. Or restore from backup if available + +# Backup checkpoint before major changes +dbutils.fs.cp( + "/checkpoints/stream", + "/checkpoints/stream_backup_20240101", + recurse=True +) +``` + +### Crash During Batch + +```python +# Scenario: Crash during batch processing +# - Latest offset = 223 (written at start) +# - Commit 223 missing (crash before finish) +# - On restart: Spark reprocesses offset 223 +# - Delta deduplication prevents duplicates (if txnVersion configured) +``` + +## Monitoring + +### Checkpoint Size + +```python +# Track checkpoint folder size +checkpoint_size = dbutils.fs.ls("/checkpoints/stream") +total_size = sum([f.size for f in checkpoint_size if f.isFile()]) +print(f"Checkpoint size: {total_size / (1024*1024):.2f} MB") + +# Alert on checkpoint access failures +try: + dbutils.fs.ls("/checkpoints/stream") +except Exception as e: + print(f"Checkpoint access failed: {e}") + # Send alert +``` + +### State Store Growth + +```python +# Monitor state store size (stateful jobs) +state_df = spark.read.format("statestore").load("/checkpoints/stream/state") + +# Check partition balance +state_df.groupBy("partitionId").count().orderBy(desc("count")).show() + +# Look for skew - one partition with 10x others = problem +# State size = f(watermark duration, key cardinality) +``` + +### Offset vs Commit Sync + +```python +# Check if offsets have matching commits +import json + +# Read latest offset +latest_offset_file = sorted(dbutils.fs.ls("/checkpoints/stream/offsets"))[-1].path +offset_data = json.loads(dbutils.fs.head(latest_offset_file)) +batch_id = latest_offset_file.split("/")[-1] + +# Check if commit exists +commit_file = f"/checkpoints/stream/commits/{batch_id}" +if dbutils.fs.exists(commit_file): + print(f"Batch {batch_id}: Committed") +else: + print(f"Batch {batch_id}: Not committed (will reprocess)") +``` + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **State growing too large** | Long watermark duration or high cardinality keys | Reduce watermark duration; reduce key cardinality | +| **Checkpoint corruption** | File system issues or manual deletion | Delete checkpoint and restart; restore from backup | +| **Slow state operations** | Partition imbalance | Check partition balance; ensure keys are evenly distributed | +| **Can't find commit file** | Normal if job crashed | Spark will reprocess on restart | +| **Offsets out of sync** | Offsets without matching commits | Indicates unprocessed batch; will reprocess | + +## Production Best Practices + +### Checkpoint Location Pattern + +```python +def get_checkpoint_path(table_name, environment="prod"): + """ + Checkpoint should be: + 1. Tied to TARGET table (not source) + 2. In persistent storage (UC Volume, S3, ADLS) + 3. Organized systematically + """ + return f"/Volumes/{environment}/checkpoints/{table_name}" + +# Usage +checkpoint = get_checkpoint_path("orders", "prod") +``` + +### Backup Strategy + +```python +# Backup checkpoint before major changes +def backup_checkpoint(checkpoint_path, backup_suffix): + backup_path = f"{checkpoint_path}_backup_{backup_suffix}" + dbutils.fs.cp(checkpoint_path, backup_path, recurse=True) + return backup_path + +# Before code changes or migrations +backup_checkpoint("/checkpoints/stream", "20240101") +``` + +### Migration + +```python +# Migrate checkpoint to new location +def migrate_checkpoint(old_path, new_path): + # Copy checkpoint folder + dbutils.fs.cp(old_path, new_path, recurse=True) + + # Update code to use new path + # Old checkpoint remains for rollback + + # Restart stream with new checkpoint location +``` + +## Production Checklist + +- [ ] Checkpoint location is persistent (S3/ADLS, not DBFS) +- [ ] Unique checkpoint per stream +- [ ] Target-tied checkpoint organization +- [ ] Backup strategy defined +- [ ] Monitoring configured (checkpoint size, access failures) +- [ ] State store growth monitored (if stateful) +- [ ] Recovery procedure documented +- [ ] Migration procedure documented + +## Related Skills + +- `kafka-to-delta` - Kafka ingestion with checkpoint management +- `stream-stream-joins` - Stateful operations and state stores +- `state-store-management` - Deep dive on state store optimization diff --git a/.claude/skills/databricks-spark-structured-streaming/kafka-streaming.md b/.claude/skills/databricks-spark-structured-streaming/kafka-streaming.md new file mode 100644 index 00000000..83630e8a --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/kafka-streaming.md @@ -0,0 +1,417 @@ +--- +name: kafka-streaming +description: Comprehensive Kafka streaming patterns including Kafka-to-Delta ingestion, Kafka-to-Kafka pipelines, and Real-Time Mode for sub-second latency. Use when building Kafka ingestion pipelines, implementing event enrichment, format transformation, or low-latency streaming workloads. +--- + +# Kafka Streaming Patterns + +Comprehensive guide to Kafka streaming with Spark Structured Streaming: ingestion to Delta, Kafka-to-Kafka pipelines, and Real-Time Mode for sub-second latency. + +## Quick Start + +### Kafka to Delta + +```python +from pyspark.sql.functions import col, from_json + +# Read from Kafka +df = (spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", "broker1:9092,broker2:9092") + .option("subscribe", "topic_name") + .option("startingOffsets", "earliest") + .option("minPartitions", "6") # Match Kafka partitions + .load() +) + +# Parse JSON value +df_parsed = df.select( + col("key").cast("string"), + from_json(col("value").cast("string"), event_schema).alias("data"), + col("topic"), col("partition"), col("offset"), + col("timestamp").alias("kafka_timestamp") +).select("key", "data.*", "topic", "partition", "offset", "kafka_timestamp") + +# Write to Delta +df_parsed.writeStream \ + .format("delta") \ + .outputMode("append") \ + .option("checkpointLocation", "/Volumes/catalog/checkpoints/kafka_stream") \ + .trigger(processingTime="30 seconds") \ + .start("/delta/bronze_events") +``` + +### Kafka to Kafka + +```python +from pyspark.sql.functions import col, from_json, to_json, struct, current_timestamp + +# Read from source Kafka +source_df = (spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", "broker1:9092") + .option("subscribe", "input-events") + .option("startingOffsets", "latest") + .load() +) + +# Parse and transform +parsed_df = source_df.select( + col("key").cast("string"), + from_json(col("value").cast("string"), event_schema).alias("data"), + col("topic").alias("source_topic") +).select("key", "data.*", "source_topic") + +# Transform events +enriched_df = parsed_df.withColumn( + "processed_at", current_timestamp() +).withColumn( + "value", to_json(struct("event_id", "user_id", "event_type", "processed_at")) +) + +# Write to output Kafka topic +enriched_df.select("key", "value").writeStream \ + .format("kafka") \ + .option("kafka.bootstrap.servers", "broker1:9092") \ + .option("topic", "output-events") \ + .option("checkpointLocation", "/checkpoints/kafka-to-kafka") \ + .trigger(processingTime="30 seconds") \ + .start() +``` + +## Common Patterns + +### Pattern 1: Bronze Layer Ingestion (Kafka to Delta) + +Minimal transformation, preserve original columns: + +```python +# Best practice: Minimal transformation, preserve original columns +# Why: Kafka retention is expensive (default 7 days) +# Delta provides permanent storage with full history + +df_bronze = (spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", servers) + .option("subscribe", topic) + .option("startingOffsets", "earliest") + .option("maxOffsetsPerTrigger", 10000) # Control batch size + .load() + .select( + col("key").cast("string"), + col("value").cast("string"), + col("topic"), col("partition"), col("offset"), + col("timestamp").alias("kafka_timestamp"), + current_timestamp().alias("ingestion_timestamp") + ) +) + +df_bronze.writeStream \ + .format("delta") \ + .outputMode("append") \ + .option("checkpointLocation", "/Volumes/catalog/checkpoints/bronze_events") \ + .trigger(processingTime="30 seconds") \ + .start("/delta/bronze_events") +``` + +### Pattern 2: Scheduled Streaming (Cost-Optimized) + +Run periodically instead of continuously: + +```python +# Run every 4 hours, not continuously +# Same code, just change trigger in job scheduler + +df_bronze.writeStream \ + .format("delta") \ + .outputMode("append") \ + .option("checkpointLocation", "/Volumes/catalog/checkpoints/bronze_events") \ + .trigger(availableNow=True) \ # Process all available, then stop + .start("/delta/bronze_events") + +# In Databricks Jobs: +# - Schedule: Every 4 hours +# - Cluster: Fixed size (no autoscaling for streaming) +# - Same streaming code, batch-style execution +``` + +### Pattern 3: Real-Time Mode (Sub-Second Latency) + +Use RTM for < 800ms latency requirements: + +```python +# Real-time trigger (Databricks 13.3+) +query = (enriched_df + .select(col("key"), col("value")) + .writeStream + .format("kafka") + .option("kafka.bootstrap.servers", brokers) + .option("topic", "output-events") + .trigger(realTime=True) # Enable RTM + .option("checkpointLocation", checkpoint_path) + .start() +) + +# RTM Cluster Requirements +spark.conf.set("spark.databricks.photon.enabled", "true") +spark.conf.set("spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider") + +# When to use RTM: +# - Latency < 800ms required +# - Photon enabled +# - Fixed-size cluster (no autoscaling) +``` + +### Pattern 4: Event Enrichment (Kafka to Kafka with Delta) + +Enrich events with dimension data: + +```python +# Read reference data (Delta table - auto-refreshed each microbatch) +user_dim = spark.table("users.dimension") + +# Stream-static join for enrichment +enriched = (parsed_df + .join(user_dim, "user_id", "left") + .withColumn("enriched_value", to_json(struct( + col("event_id"), + col("user_id"), + col("user_name"), # From dimension table + col("user_segment"), # From dimension table + col("event_type"), + col("timestamp") + ))) +) + +# Write enriched events to Kafka +enriched.select(col("key"), col("enriched_value").alias("value")).writeStream \ + .format("kafka") \ + .option("kafka.bootstrap.servers", brokers) \ + .option("topic", "enriched-events") \ + .trigger(realTime=True) \ + .option("checkpointLocation", "/checkpoints/enrichment") \ + .start() +``` + +### Pattern 5: Multi-Topic Routing + +Route events to different Kafka topics: + +```python +def route_events(batch_df, batch_id): + """Route events to different Kafka topics""" + + # High priority → urgent topic + high_priority = batch_df.filter(col("priority") == "high") + if high_priority.count() > 0: + high_priority.select("key", "value").write \ + .format("kafka") \ + .option("kafka.bootstrap.servers", brokers) \ + .option("topic", "urgent-events") \ + .save() + + # Errors → DLQ topic + errors = batch_df.filter(col("event_type") == "error") + if errors.count() > 0: + errors.select("key", "value").write \ + .format("kafka") \ + .option("kafka.bootstrap.servers", brokers) \ + .option("topic", "error-events-dlq") \ + .save() + + # All events → standard topic + batch_df.select("key", "value").write \ + .format("kafka") \ + .option("kafka.bootstrap.servers", brokers) \ + .option("topic", "standard-events") \ + .save() + +parsed_df.writeStream \ + .foreachBatch(route_events) \ + .trigger(realTime=True) \ + .option("checkpointLocation", "/checkpoints/routing") \ + .start() +``` + +### Pattern 6: Schema Validation with DLQ + +Validate schema and route invalid records: + +```python +from pyspark.sql.functions import from_json, col, lit, to_json, struct, current_timestamp + +def validate_and_route(batch_df, batch_id): + """Validate schema, route bad records to DLQ""" + + # Try to parse with strict schema + parsed = batch_df.withColumn( + "parsed", + from_json(col("value").cast("string"), validated_schema) + ) + + # Valid records + valid = parsed.filter(col("parsed").isNotNull()).select("key", "value") + + # Invalid records → DLQ + invalid = parsed.filter(col("parsed").isNull()).select( + col("key"), + to_json(struct( + col("value"), + lit("SCHEMA_VALIDATION_FAILED").alias("dlq_reason"), + current_timestamp().alias("dlq_timestamp") + )).alias("value") + ) + + # Write valid to main topic + if valid.count() > 0: + valid.write.format("kafka") \ + .option("kafka.bootstrap.servers", brokers) \ + .option("topic", "valid-events") \ + .save() + + # Write invalid to DLQ + if invalid.count() > 0: + invalid.write.format("kafka") \ + .option("kafka.bootstrap.servers", brokers) \ + .option("topic", "dlq-events") \ + .save() + +source_df.writeStream \ + .foreachBatch(validate_and_route) \ + .trigger(realTime=True) \ + .option("checkpointLocation", "/checkpoints/validation") \ + .start() +``` + +## Configuration + +### Consumer Options (Reading from Kafka) + +```python +(spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", "host1:9092,host2:9092") + .option("subscribe", "source-topic") + .option("startingOffsets", "latest") # latest, earliest, or specific JSON + .option("maxOffsetsPerTrigger", "10000") # Control batch size + .option("minPartitions", "6") # Match Kafka partitions + .option("kafka.auto.offset.reset", "latest") + .option("kafka.enable.auto.commit", "false") # Spark manages offsets + .load() +) +``` + +### Producer Options (Writing to Kafka) + +```python +(df + .select("key", "value") + .writeStream + .format("kafka") + .option("kafka.bootstrap.servers", "host1:9092,host2:9092") + .option("topic", "target-topic") + .option("kafka.acks", "all") # Durability: all, 1, 0 + .option("kafka.retries", "3") + .option("kafka.batch.size", "16384") + .option("kafka.linger.ms", "5") + .option("kafka.compression.type", "lz4") # lz4, snappy, gzip + .option("checkpointLocation", checkpoint_path) + .start() +) +``` + +### Security (SASL/SSL) + +```python +# Using Databricks secrets +kafka_username = dbutils.secrets.get("kafka-scope", "username") +kafka_password = dbutils.secrets.get("kafka-scope", "password") + +# SASL/PLAIN Authentication +df.writeStream \ + .format("kafka") \ + .option("kafka.bootstrap.servers", brokers) \ + .option("topic", target_topic) \ + .option("kafka.security.protocol", "SASL_SSL") \ + .option("kafka.sasl.mechanism", "PLAIN") \ + .option("kafka.sasl.jaas.config", + f'org.apache.kafka.common.security.plain.PlainLoginModule required username="{kafka_username}" password="{kafka_password}";') \ + .option("checkpointLocation", checkpoint_path) \ + .start() +``` + +## Performance Tuning + +| Parameter | Recommendation | Why | +|-----------|---------------|-----| +| minPartitions | Match Kafka partitions | Optimal parallelism | +| maxOffsetsPerTrigger | 10,000-100,000 | Balance latency vs throughput | +| trigger interval | Business SLA / 3 | Recovery time buffer | +| RTM | Only if < 800ms required | Microbatch more cost-effective | + +## Monitoring + +### Key Metrics + +```python +# Programmatic monitoring +for stream in spark.streams.active: + progress = stream.lastProgress + if progress: + print(f"Input rate: {progress.get('inputRowsPerSecond', 0)} rows/sec") + print(f"Processing rate: {progress.get('processedRowsPerSecond', 0)} rows/sec") + + # Kafka-specific metrics + sources = progress.get("sources", []) + for source in sources: + end_offset = source.get("endOffset", {}) + latest_offset = source.get("latestOffset", {}) + + # Calculate lag per partition + for topic, partitions in end_offset.items(): + for partition, end in partitions.items(): + latest = latest_offset.get(topic, {}).get(partition, end) + lag = int(latest) - int(end) + print(f"Topic {topic}, Partition {partition}: Lag = {lag}") +``` + +### Spark UI Checks + +- **Input Rate vs Processing Rate**: Processing must be > Input +- **Max Offsets Behind Latest**: Should be consistent or dropping +- **Batch Duration**: Should be < trigger interval + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **No data being read** | `startingOffsets` default is "latest" | Use "earliest" for existing data | +| **High latency** | Microbatch overhead | Use RTM (trigger(realTime=True)) | +| **Consumer lag** | Processing < Input rate | Scale cluster; reduce maxOffsetsPerTrigger | +| **Duplicate messages** | Exactly-once not configured | Enable idempotent producer (acks=all) | +| **Falling behind** | Processing < Input rate | Increase cluster size | +| **Can't use autoscaling** | Streaming requirement | Use fixed-size clusters | + +## Production Checklist + +- [ ] Checkpoint location is persistent (UC volumes, not DBFS) +- [ ] Unique checkpoint per pipeline +- [ ] Fixed-size cluster (no autoscaling for streaming/RTM) +- [ ] RTM enabled only if latency < 800ms required +- [ ] Consumer lag monitored and alerts configured +- [ ] Producer acks=all for durability +- [ ] Schema validation with DLQ configured +- [ ] Security (SASL/SSL) configured for production +- [ ] Exactly-once semantics verified + +## Related Skills + +- `stream-static-joins` - Enrichment patterns with Delta tables +- `stream-stream-joins` - Event correlation across Kafka topics +- `checkpoint-best-practices` - Checkpoint configuration +- `trigger-tuning` - Trigger configuration and RTM setup diff --git a/.claude/skills/databricks-spark-structured-streaming/merge-operations.md b/.claude/skills/databricks-spark-structured-streaming/merge-operations.md new file mode 100644 index 00000000..374239ad --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/merge-operations.md @@ -0,0 +1,358 @@ +--- +name: merge-operations +description: Comprehensive guide to Delta MERGE operations in streaming including performance optimization, parallel merges, and Liquid Clustering configuration. Use when implementing upserts, optimizing merge performance, performing parallel merges to multiple tables, or eliminating optimize pauses. +--- + +# Merge Operations in Streaming + +Comprehensive guide to Delta MERGE operations: performance optimization, parallel merges to multiple tables, and modern Delta features (Liquid Clustering + Deletion Vectors + Row-Level Concurrency). + +## Quick Start + +### Basic MERGE with Optimization + +```python +from delta.tables import DeltaTable + +# Enable modern Delta features +spark.sql(""" + ALTER TABLE target_table SET TBLPROPERTIES ( + 'delta.enableDeletionVectors' = true, + 'delta.enableRowLevelConcurrency' = true, + 'delta.liquid.clustering' = true + ) +""") + +# MERGE in ForEachBatch +def upsert_batch(batch_df, batch_id): + batch_df.createOrReplaceTempView("updates") + spark.sql(""" + MERGE INTO target_table t + USING updates s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET * + WHEN NOT MATCHED THEN INSERT * + """) + # No optimize needed - Liquid Clustering handles it automatically + +stream.writeStream \ + .foreachBatch(upsert_batch) \ + .option("checkpointLocation", "/checkpoints/merge") \ + .start() +``` + +### Parallel MERGE to Multiple Tables + +```python +from delta.tables import DeltaTable +from concurrent.futures import ThreadPoolExecutor, as_completed + +def parallel_merge_multiple_tables(batch_df, batch_id): + """Merge into multiple tables in parallel""" + + batch_df.cache() + + def merge_table(table_name, merge_key): + target = DeltaTable.forName(spark, table_name) + source = batch_df.alias("source") + + (target.alias("target") + .merge(source, f"target.{merge_key} = source.{merge_key}") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute() + ) + return f"Merged {table_name}" + + tables = [ + ("silver.customers", "customer_id"), + ("silver.orders", "order_id"), + ("silver.products", "product_id") + ] + + # Parallel merges + with ThreadPoolExecutor(max_workers=3) as executor: + futures = { + executor.submit(merge_table, table_name, merge_key): table_name + for table_name, merge_key in tables + } + + for future in as_completed(futures): + future.result() # Raise on error + + batch_df.unpersist() + +stream.writeStream \ + .foreachBatch(parallel_merge_multiple_tables) \ + .option("checkpointLocation", "/checkpoints/parallel_merge") \ + .start() +``` + +## Core Concepts + +### Liquid Clustering + DV + RLC + +Enable modern Delta features for optimal merge performance: + +```sql +-- Enable for target table +ALTER TABLE target_table SET TBLPROPERTIES ( + 'delta.enableDeletionVectors' = true, + 'delta.enableRowLevelConcurrency' = true, + 'delta.liquid.clustering' = true +); +``` + +**Benefits:** +- **Deletion Vectors**: Soft deletes without file rewrite +- **Row-Level Concurrency**: Concurrent updates to different rows +- **Liquid Clustering**: Automatic optimization without pauses +- **Result**: Eliminates optimize pauses, lower P99 latency, simpler code + +## Common Patterns + +### Pattern 1: Basic MERGE with Optimization + +```python +def optimized_merge(batch_df, batch_id): + """MERGE with optimized table""" + batch_df.createOrReplaceTempView("updates") + + spark.sql(""" + MERGE INTO target_table t + USING updates s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET * + WHEN NOT MATCHED THEN INSERT * + """) + # No optimize needed - Liquid Clustering handles it + +stream.writeStream \ + .foreachBatch(optimized_merge) \ + .option("checkpointLocation", "/checkpoints/merge") \ + .start() +``` + +### Pattern 2: Parallel MERGE to Multiple Tables + +```python +from concurrent.futures import ThreadPoolExecutor, as_completed + +def parallel_merge(batch_df, batch_id): + """Merge into multiple tables in parallel""" + + batch_df.cache() + + def merge_one_table(table_name, merge_key): + target = DeltaTable.forName(spark, table_name) + source = batch_df.alias("source") + + (target.alias("target") + .merge(source, f"target.{merge_key} = source.{merge_key}") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute() + ) + return table_name + + tables = [ + ("silver.customers", "customer_id"), + ("silver.orders", "order_id"), + ("silver.products", "product_id") + ] + + # Optimal thread count: min(number_of_tables, cluster_cores / 2) + max_workers = min(len(tables), max(2, total_cores // 2)) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit(merge_one_table, table_name, merge_key): table_name + for table_name, merge_key in tables + } + + errors = [] + for future in as_completed(futures): + table_name = futures[future] + try: + future.result() + except Exception as e: + errors.append((table_name, str(e))) + + batch_df.unpersist() + + if errors: + raise Exception(f"Merge failures: {errors}") +``` + +### Pattern 3: MERGE with Partition Pruning + +```python +def partition_pruned_merge(batch_df, batch_id): + """MERGE with partition column in condition""" + batch_df.createOrReplaceTempView("updates") + + # Include partition column in merge condition + spark.sql(""" + MERGE INTO target_table t + USING updates s + ON t.id = s.id AND t.date = s.date -- partition column + WHEN MATCHED THEN UPDATE SET * + WHEN NOT MATCHED THEN INSERT * + """) + # Skips irrelevant partitions for faster execution +``` + +### Pattern 4: CDC Multi-Target with Parallel MERGE + +```python +def cdc_parallel_merge(batch_df, batch_id): + """Apply CDC changes to multiple tables in parallel""" + + batch_df.cache() + + # Split by operation type + deletes = batch_df.filter(col("_op") == "DELETE") + upserts = batch_df.filter(col("_op").isin(["INSERT", "UPDATE"])) + + def merge_cdc_table(table_name, merge_key): + target = DeltaTable.forName(spark, table_name) + + # Upserts + if upserts.count() > 0: + (target.alias("target") + .merge(upserts.alias("source"), f"target.{merge_key} = source.{merge_key}") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute() + ) + + # Deletes + if deletes.count() > 0: + (target.alias("target") + .merge(deletes.alias("source"), f"target.{merge_key} = source.{merge_key}") + .whenMatchedDelete() + .execute() + ) + + tables = [ + ("silver.customers", "customer_id"), + ("silver.orders", "order_id") + ] + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = { + executor.submit(merge_cdc_table, table_name, merge_key): table_name + for table_name, merge_key in tables + } + + for future in as_completed(futures): + future.result() + + batch_df.unpersist() +``` + +## Performance Optimization + +### Enable Liquid Clustering + DV + RLC + +```sql +-- Create table with Liquid Clustering +CREATE TABLE target_table ( + id STRING, + name STRING, + updated_at TIMESTAMP +) USING DELTA +CLUSTER BY (id) +TBLPROPERTIES ( + 'delta.enableDeletionVectors' = true, + 'delta.enableRowLevelConcurrency' = true +); + +-- Or alter existing table +ALTER TABLE target_table SET TBLPROPERTIES ( + 'delta.enableDeletionVectors' = true, + 'delta.enableRowLevelConcurrency' = true, + 'delta.liquid.clustering' = true +); +ALTER TABLE target_table CLUSTER BY (id); +``` + +### Z-Ordering on Merge Key + +```sql +-- Z-Order on merge key for faster lookups +OPTIMIZE target_table ZORDER BY (id); + +-- Run periodically or via Predictive Optimization +-- 5-10x faster for targeted lookups +``` + +### File Size Tuning + +```sql +-- Target file size for optimal merge +ALTER TABLE target_table SET TBLPROPERTIES ( + 'delta.targetFileSize' = '128mb' +); +``` + +### Optimal Thread Count + +```python +# Formula: min(number_of_tables, cluster_cores / 2) +# Example: 4 tables, 8 cores → 4 workers +# Example: 2 tables, 4 cores → 2 workers + +max_workers = min(len(tables), max(2, total_cores // 2)) +``` + +## Monitoring + +### Track Merge Performance + +```python +import time + +def monitored_merge(batch_df, batch_id): + start_time = time.time() + + batch_df.createOrReplaceTempView("updates") + spark.sql(""" + MERGE INTO target_table t + USING updates s ON t.id = s.id + WHEN MATCHED THEN UPDATE SET * + WHEN NOT MATCHED THEN INSERT * + """) + + duration = time.time() - start_time + print(f"Merge duration: {duration:.2f}s") + + # Alert if duration exceeds threshold + if duration > 30: + print(f"WARNING: Merge duration {duration:.2f}s exceeds threshold") +``` + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **High P99 latency** | OPTIMIZE pauses | Enable Liquid Clustering (no pauses) | +| **Merge conflicts** | Concurrent updates to same rows | Enable Row-Level Concurrency | +| **Slow merges** | Large files, no optimization | Enable Liquid Clustering; Z-Order on merge key | +| **Too many threads** | Resource contention | Reduce max_workers; match to cluster capacity | +| **Partial failures** | One merge fails | Collect all errors; fail batch if any error | + +## Production Checklist + +- [ ] Liquid Clustering + DV + RLC enabled on all target tables +- [ ] Z-Ordering configured on merge keys +- [ ] Optimal thread count configured (start with 2) +- [ ] Error handling implemented (collect all errors) +- [ ] Performance monitoring per table +- [ ] Cache used to avoid recomputation +- [ ] Unpersist after writes +- [ ] File size tuned (128MB target) + +## Related Skills + +- `multi-sink-writes` - Multi-sink write patterns +- `partitioning-strategy` - Partition optimization for merges +- `checkpoint-best-practices` - Checkpoint configuration diff --git a/.claude/skills/databricks-spark-structured-streaming/multi-sink-writes.md b/.claude/skills/databricks-spark-structured-streaming/multi-sink-writes.md new file mode 100644 index 00000000..6611ab0d --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/multi-sink-writes.md @@ -0,0 +1,427 @@ +--- +name: multi-sink-writes +description: Write a single Spark stream to multiple Delta tables or Kafka topics using ForEachBatch. Use when fanning out streaming data to multiple sinks, implementing medallion architecture (bronze/silver/gold), conditional routing, CDC patterns, or creating materialized views from a single stream. +--- + +# Multi-Sink Writes + +Write a single streaming source to multiple Delta tables or Kafka topics efficiently using ForEachBatch. Read once, write many - avoiding reprocessing the source multiple times. + +## Quick Start + +```python +from pyspark.sql.functions import col, current_timestamp + +def write_multiple_tables(batch_df, batch_id): + """Write batch to multiple sinks""" + # Bronze - raw data + batch_df.write \ + .format("delta") \ + .mode("append") \ + .option("txnVersion", batch_id) \ + .option("txnAppId", "multi_sink_job") \ + .save("/delta/bronze_events") + + # Silver - cleansed + cleansed = batch_df.dropDuplicates(["event_id"]) + cleansed.write \ + .format("delta") \ + .mode("append") \ + .option("txnVersion", batch_id) \ + .option("txnAppId", "multi_sink_job_silver") \ + .save("/delta/silver_events") + + # Gold - aggregated + aggregated = batch_df.groupBy("category").count() + aggregated.write \ + .format("delta") \ + .mode("append") \ + .option("txnVersion", batch_id) \ + .option("txnAppId", "multi_sink_job_gold") \ + .save("/delta/category_counts") + +stream.writeStream \ + .foreachBatch(write_multiple_tables) \ + .option("checkpointLocation", "/checkpoints/multi_sink") \ + .start() +``` + +## Core Concepts + +### One Source, One Checkpoint + +Use a single checkpoint for the entire multi-sink stream: + +```python +# CORRECT: One checkpoint for all sinks +stream.writeStream \ + .foreachBatch(multi_sink_function) \ + .option("checkpointLocation", "/checkpoints/single_source_multi_sink") \ + .start() + +# WRONG: Don't create separate streams +# Each stream would reprocess the source independently +``` + +### Transactional Guarantees + +Each ForEachBatch call represents one epoch. All writes within the batch: +- See the same input data +- Share the same batch_id +- Are idempotent if using txnVersion + +## Common Patterns + +### Pattern 1: Bronze-Silver-Gold Medallion Architecture + +Single stream feeding all three medallion layers: + +```python +from pyspark.sql.functions import window, count, sum, current_timestamp + +def medallion_architecture(batch_df, batch_id): + """Single stream feeding all three medallion layers""" + + # Bronze: Raw ingestion + (batch_df.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "medallion_bronze") + .saveAsTable("bronze.events") + ) + + # Silver: Cleansed and validated + silver_df = (batch_df + .dropDuplicates(["event_id"]) + .filter(col("status").isin(["active", "pending"])) + .withColumn("processed_at", current_timestamp()) + ) + + (silver_df.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "medallion_silver") + .saveAsTable("silver.events") + ) + + # Gold: Business aggregates + gold_df = (silver_df + .groupBy(window(col("timestamp"), "5 minutes"), "category") + .agg( + count("*").alias("event_count"), + sum("amount").alias("total_amount") + ) + ) + + (gold_df.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "medallion_gold") + .saveAsTable("gold.category_metrics") + ) + +stream.writeStream \ + .foreachBatch(medallion_architecture) \ + .trigger(processingTime="30 seconds") \ + .option("checkpointLocation", "/checkpoints/medallion") \ + .start() +``` + +### Pattern 2: Conditional Routing + +Route events to different tables based on criteria: + +```python +def route_by_type(batch_df, batch_id): + """Route events to different tables based on type""" + + # Split by event type + orders = batch_df.filter(col("event_type") == "order") + refunds = batch_df.filter(col("event_type") == "refund") + reviews = batch_df.filter(col("event_type") == "review") + + # Write to respective tables + if orders.count() > 0: + (orders.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "router_orders") + .saveAsTable("orders") + ) + + if refunds.count() > 0: + (refunds.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "router_refunds") + .saveAsTable("refunds") + ) + + if reviews.count() > 0: + (reviews.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "router_reviews") + .saveAsTable("reviews") + ) +``` + +### Pattern 3: Parallel Fan-Out + +Write to multiple sinks in parallel for independent tables: + +```python +from concurrent.futures import ThreadPoolExecutor, as_completed + +def parallel_write(batch_df, batch_id): + """Write to multiple sinks in parallel""" + + # Cache to avoid recomputation + batch_df.cache() + + def write_table(table_name, filter_expr=None): + """Write filtered data to table""" + df = batch_df.filter(filter_expr) if filter_expr else batch_df + (df.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", f"parallel_{table_name}") + .saveAsTable(table_name) + ) + return f"Wrote {table_name}" + + # Define tables and filters + tables = [ + ("bronze.all_events", None), + ("silver.errors", col("level") == "ERROR"), + ("silver.warnings", col("level") == "WARN"), + ("gold.metrics", col("type") == "metric") + ] + + # Parallel writes + with ThreadPoolExecutor(max_workers=4) as executor: + futures = { + executor.submit(write_table, table_name, filter_expr): table_name + for table_name, filter_expr in tables + } + + errors = [] + for future in as_completed(futures): + table_name = futures[future] + try: + future.result() + except Exception as e: + errors.append((table_name, str(e))) + + batch_df.unpersist() + + if errors: + raise Exception(f"Write failures: {errors}") +``` + +### Pattern 4: Materialized Views + +Create multiple derived views from the same stream: + +```python +from pyspark.sql.functions import window, count, sum + +def create_materialized_views(batch_df, batch_id): + """Create multiple derived views from the same stream""" + + # Base: All events + (batch_df.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "views_raw") + .save("/delta/views/raw") + ) + + # View 1: Hourly aggregations + hourly = (batch_df + .withWatermark("event_time", "1 hour") + .groupBy(window(col("event_time"), "1 hour"), col("category")) + .agg( + count("*").alias("event_count"), + sum("value").alias("total_value") + ) + ) + + (hourly.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "views_hourly") + .save("/delta/views/hourly") + ) + + # View 2: User sessions (15 min window) + sessions = (batch_df + .withWatermark("event_time", "15 minutes") + .groupBy(window(col("event_time"), "15 minutes"), col("user_id")) + .agg(count("*").alias("actions")) + ) + + (sessions.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "views_sessions") + .save("/delta/views/sessions") + ) +``` + +### Pattern 5: Error Handling with Dead Letter Queue + +Route invalid records to DLQ: + +```python +from pyspark.sql.functions import when, lit + +def write_with_dlq(batch_df, batch_id): + """Write valid records to target, invalid to dead letter queue""" + + # Validation + valid = batch_df.filter( + col("required_field").isNotNull() & + col("timestamp").isNotNull() + ) + invalid = batch_df.filter( + col("required_field").isNull() | + col("timestamp").isNull() + ) + + # Write valid data + if valid.count() > 0: + (valid.write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "multi_sink_valid") + .saveAsTable("silver.valid_events") + ) + + # Write invalid to DLQ with metadata + if invalid.count() > 0: + dlq_df = (invalid + .withColumn("_error_reason", + when(col("required_field").isNull(), "missing_required_field") + .otherwise("missing_timestamp")) + .withColumn("_batch_id", lit(batch_id)) + .withColumn("_processed_at", current_timestamp()) + ) + + (dlq_df.write + .format("delta") + .mode("append") + .saveAsTable("errors.dead_letter_queue") + ) +``` + +## Performance Optimization + +### Minimize Recomputation + +Cache the batch DataFrame to avoid recomputation: + +```python +def optimized_multi_sink(batch_df, batch_id): + """Cache to avoid recomputation""" + + # Cache the batch + batch_df.cache() + + # Multiple writes from cached data + batch_df.write... # Sink 1 + batch_df.filter(...).write... # Sink 2 + batch_df.filter(...).write... # Sink 3 + + # Unpersist when done + batch_df.unpersist() +``` + +### Parallel Writes + +Use ThreadPoolExecutor for independent writes: + +```python +from concurrent.futures import ThreadPoolExecutor + +def parallel_write(batch_df, batch_id): + """Write to independent tables in parallel""" + + batch_df.cache() + + def write_table(table_name, df): + df.write.format("delta").mode("append").saveAsTable(table_name) + + # Parallel writes + with ThreadPoolExecutor(max_workers=4) as executor: + executor.submit(write_table, "table1", batch_df) + executor.submit(write_table, "table2", batch_df.filter(...)) + executor.submit(write_table, "table3", batch_df.filter(...)) + + batch_df.unpersist() +``` + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **Slow writes** | Sequential processing | Use parallel ThreadPoolExecutor | +| **Recomputation** | Multiple actions on same DataFrame | Cache the batch DataFrame | +| **Partial failures** | One sink fails | Use idempotent writes; Spark retries entire batch | +| **Schema conflicts** | Tables have different schemas | Transform before each write | +| **Resource contention** | Too many concurrent writes | Limit parallelism; batch writes | + +## Production Best Practices + +### Idempotent Writes + +Always use txnVersion with batch_id: + +```python +.write + .format("delta") + .option("txnVersion", batch_id) + .option("txnAppId", "unique_app_id_per_table") + .mode("append") +``` + +### Keep Batch Processing Fast + +```python +# GOOD: Simple filters and writes +def efficient_write(df, batch_id): + df.filter(...).write.save("/delta/table1") + df.filter(...).write.save("/delta/table2") + +# BAD: Expensive aggregations (move to stream definition) +def inefficient_write(df, batch_id): + df.groupBy(...).agg(...).write.save("/delta/table3") # Move to stream! +``` + +## Production Checklist + +- [ ] One checkpoint per multi-sink stream +- [ ] Idempotent writes configured (txnVersion/txnAppId) +- [ ] Cache used to avoid recomputation +- [ ] Parallel writes for independent tables +- [ ] Error handling and DLQ configured +- [ ] Schema evolution handled +- [ ] Performance monitoring per sink + +## Related Skills + +- `merge-operations` - Parallel MERGE operations +- `kafka-streaming` - Kafka ingestion patterns +- `stream-static-joins` - Enrichment before multi-sink writes +- `checkpoint-best-practices` - Checkpoint configuration diff --git a/.claude/skills/databricks-spark-structured-streaming/stateful-operations.md b/.claude/skills/databricks-spark-structured-streaming/stateful-operations.md new file mode 100644 index 00000000..625f53eb --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/stateful-operations.md @@ -0,0 +1,397 @@ +--- +name: stateful-operations +description: Configure watermarks and manage state stores for Spark Structured Streaming stateful operations. Use when setting up stateful operations, tuning watermark duration, handling late-arriving data, configuring RocksDB for large state, monitoring state store size, or optimizing state performance. +--- + +# Stateful Operations: Watermarks and State Stores + +Configure watermarks to handle late-arriving data and manage state stores for stateful streaming operations. Watermarks control state cleanup, while state stores handle the storage and retrieval of stateful data. + +## Quick Start + +```python +# Enable RocksDB for large state stores +spark.conf.set( + "spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider" +) + +# Stateful operation with watermark +df = (spark.readStream + .format("kafka") + .option("subscribe", "events") + .load() + .select(from_json(col("value").cast("string"), schema).alias("data")) + .select("data.*") + .withWatermark("event_time", "10 minutes") # Late data threshold + state cleanup + .dropDuplicates(["event_id"]) # Stateful operation +) + +# Watermark = latest_event_time - 10 minutes +# State automatically expires after watermark duration +``` + +## Watermark Configuration + +### How Watermarks Work + +```python +# Watermark = latest_event_time - delay_threshold +.withWatermark("event_time", "10 minutes") + +# Events with timestamp < watermark are considered "too late" +# State for late events is automatically cleaned up +# Late events may be dropped (outer joins) or processed (inner joins) +``` + +### Watermark Duration Selection + +| Watermark Setting | Effect | Use Case | +|-------------------|--------|----------| +| `"10 minutes"` | Moderate latency | General streaming | +| `"1 hour"` | High completeness | Financial transactions | +| `"5 minutes"` | Low latency | Real-time analytics | +| `"24 hours"` | Batch-like | Backfill scenarios | + +**Rule of thumb**: Start with 2-3× your p95 latency. Monitor late data rate and adjust. + +### Watermark and State Size + +```python +# Watermark directly affects state store size +# State kept for watermark duration + processing time + +# Example calculation: +# - 10 minute watermark +# - 1M events/min +# - State size = ~10M keys × key_size + +# Reduce watermark to reduce state size +.withWatermark("event_time", "5 minutes") # Smaller state + +# State automatically expires after watermark duration +# No manual cleanup needed +``` + +## State Store Configuration + +### Enable RocksDB + +Use RocksDB for state stores exceeding memory capacity: + +```python +# Enable RocksDB state store provider +spark.conf.set( + "spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider" +) + +# Benefits: +# - State stored on disk, reducing memory pressure +# - Recommended for: High cardinality keys, long watermark durations +# - Better performance for large state stores +``` + +### State Store Configuration + +```python +# State store batch retention +spark.conf.set("spark.sql.streaming.stateStore.minBatchesToRetain", "2") + +# State maintenance interval +spark.conf.set("spark.sql.streaming.stateStore.maintenanceInterval", "5m") + +# State store location (default: checkpoint/state) +# Automatically managed by Spark +``` + +## Common Patterns + +### Pattern 1: Basic Stateful Operation with Watermark + +```python +# Watermark for deduplication +df = (spark.readStream + .format("kafka") + .option("subscribe", "events") + .load() + .select(from_json(col("value").cast("string"), schema).alias("data")) + .select("data.*") + .withWatermark("event_time", "10 minutes") + .dropDuplicates(["event_id"]) +) + +# State expires after watermark duration +# Prevents infinite state growth +``` + +### Pattern 2: Join-Specific Watermark Tuning + +Different watermarks for streams with different latencies: + +```python +# Fast source: 5 minute watermark +impressions = (spark.readStream + .format("kafka") + .option("subscribe", "impressions") + .load() + .select(from_json(col("value").cast("string"), impression_schema).alias("data")) + .select("data.*") + .withWatermark("impression_time", "5 minutes") +) + +# Slower source: 15 minute watermark +clicks = (spark.readStream + .format("kafka") + .option("subscribe", "clicks") + .load() + .select(from_json(col("value").cast("string"), click_schema).alias("data")) + .select("data.*") + .withWatermark("click_time", "15 minutes") +) + +# Effective watermark = max(5, 15) = 15 minutes +joined = impressions.join( + clicks, + expr(""" + impressions.ad_id = clicks.ad_id AND + clicks.click_time BETWEEN impressions.impression_time AND + impressions.impression_time + interval 1 hour + """), + "inner" +) +``` + +### Pattern 3: Windowed Aggregations with Watermark + +```python +from pyspark.sql.functions import window, count, sum, max, current_timestamp + +windowed = (df + .withWatermark("event_time", "10 minutes") + .groupBy( + window(col("event_time"), "5 minutes"), + col("user_id") + ) + .agg( + count("*").alias("event_count"), + sum("value").alias("total_value"), + max("event_time").alias("latest_event") + ) + .withColumn("processing_time", current_timestamp()) +) + +# Use update mode for corrected results when late data arrives +windowed.writeStream \ + .outputMode("update") \ + .format("delta") \ + .option("checkpointLocation", "/checkpoints/windowed") \ + .start("/delta/windowed_metrics") +``` + +### Pattern 4: Monitor State Partition Balance + +Check for state store skew: + +```python +def check_state_balance(checkpoint_path): + """Check state store partition balance""" + state_df = spark.read.format("statestore").load(f"{checkpoint_path}/state") + + partition_counts = state_df.groupBy("partitionId").count().orderBy(desc("count")) + partition_counts.show() + + # Calculate skew + counts = [row['count'] for row in partition_counts.collect()] + if counts: + max_count = max(counts) + min_count = min(counts) + skew_ratio = max_count / min_count if min_count > 0 else float('inf') + + print(f"State skew ratio: {skew_ratio:.2f}") + if skew_ratio > 10: + print("WARNING: High state skew detected") + return False + return True +``` + +### Pattern 5: Monitor State Growth + +```python +def monitor_state_growth(checkpoint_path): + """Track state store growth""" + state_df = spark.read.format("statestore").load(f"{checkpoint_path}/state") + + # Current state size + total_rows = state_df.count() + + print(f"State rows: {total_rows}") + + # Check expiration + from pyspark.sql.functions import current_timestamp, col + expired = state_df.filter(col("expirationMs") < current_timestamp().cast("long") * 1000) + expired_count = expired.count() + + print(f"Expired state rows: {expired_count}") + print(f"Active state rows: {total_rows - expired_count}") +``` + +## State Size Control + +### Use Watermarks + +Watermarks automatically clean up expired state: + +```python +# State expires after watermark duration +.withWatermark("event_time", "10 minutes") + +# State size = f(watermark duration, key cardinality) +# 10 min watermark × 1M events/min = manageable +# 72 hour watermark × 1M events/min = very large +``` + +### Reduce Key Cardinality + +```python +# Bad: High cardinality keys +.dropDuplicates(["user_id"]) # Millions of distinct values + +# Good: Lower cardinality or expiring keys +.dropDuplicates(["session_id"]) # Sessions expire naturally +.dropDuplicates(["event_id", "date"]) # Partition by date reduces cardinality +``` + +## Monitoring + +### Programmatic State Monitoring + +```python +# Monitor state size programmatically +for stream in spark.streams.active: + progress = stream.lastProgress + + if progress and "stateOperators" in progress: + for op in progress["stateOperators"]: + print(f"Operator: {op.get('operatorName', 'unknown')}") + print(f"State rows: {op.get('numRowsTotal', 0)}") + print(f"State memory: {op.get('memoryUsedBytes', 0)}") + print(f"State on disk: {op.get('diskBytesUsed', 0)}") +``` + +### Track Late Data Rates + +```python +# Monitor late data impact +late_data_stats = spark.sql(""" + SELECT + date_trunc('hour', event_time) as hour, + COUNT(*) as total_events, + SUM(CASE + WHEN unix_timestamp(processing_time) - unix_timestamp(event_time) > 600 + THEN 1 ELSE 0 + END) as late_events, + AVG(unix_timestamp(processing_time) - unix_timestamp(event_time)) as avg_delay_seconds, + MAX(unix_timestamp(processing_time) - unix_timestamp(event_time)) as max_delay_seconds + FROM events + WHERE processing_time >= current_timestamp() - interval 24 hours + GROUP BY 1 + ORDER BY 1 DESC +""") +``` + +## Late Data Classification + +| Delay | Category | Handling | +|-------|----------|----------| +| < Watermark | On-time | Normal processing | +| Watermark < delay < 2×Watermark | Late | Join with inner match; may still process | +| > 2×Watermark | Very late | DLQ for manual handling | + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **State store explosion** | Watermark too long | Reduce watermark; archive old state | +| **Late data dropped** | Watermark too short | Increase watermark; analyze latency patterns | +| **State too large** | High cardinality keys or long watermark | Reduce key cardinality; decrease watermark duration | +| **State partition skew** | Uneven key distribution | Ensure keys are evenly distributed; consider salting | +| **OOM errors** | State exceeds memory | Enable RocksDB; increase memory; reduce watermark | +| **State not expiring** | Watermark not configured | Add watermark to stateful operations | + +## State Store Recovery + +```python +# Scenario 1: State store corruption +# Solution: Delete state folder, restart stream +# State will rebuild from watermark + +dbutils.fs.rm("/checkpoints/stream/state", recurse=True) + +# Restart stream - state rebuilds automatically +# Note: May reprocess some data within watermark window + +# Scenario 2: State store too large +# Solution: Reduce watermark duration +.withWatermark("event_time", "5 minutes") # Reduced from 10 minutes + +# Scenario 3: State partition imbalance +# Solution: Ensure keys are evenly distributed +# Consider salting keys if needed +``` + +## Production Best Practices + +### Always Use Watermarks for Stateful Operations + +```python +# REQUIRED: Watermark for stateful operations +df.withWatermark("event_time", "10 minutes").dropDuplicates(["id"]) + +# REQUIRED: Watermark for aggregations +df.withWatermark("event_time", "10 minutes").groupBy(...).agg(...) + +# REQUIRED: Watermark for stream-stream joins +stream1.withWatermark("ts", "10 min").join(stream2.withWatermark("ts", "10 min")) +``` + +### Watermark Selection + +```python +# Rule of thumb: 2-3× p95 latency +# Example: p95 latency = 5 minutes → watermark = 10-15 minutes + +# Start conservative, adjust based on monitoring +.withWatermark("event_time", "10 minutes") # Start here +# Monitor late data rate +# Increase if too many late events +# Decrease if state too large +``` + +### Use RocksDB for Large State + +```python +# Enable RocksDB if state > memory capacity +# Typical threshold: > 100M keys or > 10GB state + +spark.conf.set( + "spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider" +) +``` + +## Production Checklist + +- [ ] Watermark configured for all stateful operations +- [ ] Watermark duration matches latency requirements (2-3× p95) +- [ ] RocksDB enabled for large state stores +- [ ] State size monitored and alerts configured +- [ ] State partition balance checked regularly +- [ ] State growth tracked over time +- [ ] Late data monitoring configured +- [ ] Recovery procedure documented + +## Related Skills + +- `stream-stream-joins` - Late data in joins +- `checkpoint-best-practices` - Checkpoint and state recovery diff --git a/.claude/skills/databricks-spark-structured-streaming/stream-static-joins.md b/.claude/skills/databricks-spark-structured-streaming/stream-static-joins.md new file mode 100644 index 00000000..614d87c8 --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/stream-static-joins.md @@ -0,0 +1,519 @@ +--- +name: stream-static-joins +description: Enrich streaming data with Delta dimension tables in real-time. Use when joining fast-moving streaming events with slowly-changing reference data (device dimensions, user profiles, product catalogs), implementing real-time data enrichment, or adding context to streaming events without state management overhead. +--- + +# Stream-Static Joins + +Enrich streaming data with slowly-changing reference data stored in Delta tables. Stream-static joins are stateless and automatically refresh dimension data each microbatch. + +## Quick Start + +```python +from pyspark.sql.functions import col, from_json + +# Streaming source (IoT events from Kafka) +iot_stream = (spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", "broker:9092") + .option("subscribe", "iot-events") + .load() + .select(from_json(col("value").cast("string"), event_schema).alias("data")) + .select("data.*") +) + +# Static Delta dimension table (refreshes each microbatch) +device_dim = spark.table("device_dimensions") + +# Enrich streaming data with left join (recommended) +enriched = iot_stream.join( + device_dim, + "device_id", + "left" # Preserves all streaming events +).select( + iot_stream["*"], + device_dim["device_type"], + device_dim["location"], + device_dim["manufacturer"], + device_dim["updated_at"].alias("dim_updated_at") +) + +# Write enriched data +query = (enriched + .writeStream + .format("delta") + .outputMode("append") + .option("checkpointLocation", "/Volumes/catalog/checkpoints/enriched_events") + .trigger(processingTime="30 seconds") + .start("/delta/enriched_iot_events") +) +``` + +## Core Concepts + +### Why Delta Tables Matter + +Delta tables enable automatic version checking each microbatch: + +```python +# Delta table: Version checked every microbatch +device_dim = spark.table("device_dimensions") # Reads latest version automatically + +# Non-Delta format: Read once at startup (truly static) +device_dim = spark.read.parquet("/path/to/devices") # No refresh +``` + +**Key Insight**: Delta's versioning ensures each microbatch gets the latest dimension data without manual refresh. + +### Join Types and Production Use + +| Join Type | Behavior | Production Use | +|-----------|----------|----------------| +| **Left** | Preserves all stream events | ✅ Recommended - prevents data loss | +| **Inner** | Drops unmatched events | ⚠️ Risk of data loss - avoid in production | +| **Right** | Preserves all dimension rows | Rarely used | +| **Full** | Preserves both sides | Rarely used | + +**Production Rule**: Always use left join to prevent dropping valid streaming events. + +## Common Patterns + +### Pattern 1: Basic Device Enrichment + +Enrich IoT events with device metadata: + +```python +# Streaming IoT events +iot_stream = (spark + .readStream + .format("kafka") + .option("subscribe", "iot-events") + .load() + .select(from_json(col("value").cast("string"), event_schema).alias("data")) + .select("data.*") +) + +# Device dimension table +device_dim = spark.table("device_dimensions") + +# Left join to preserve all events +enriched = iot_stream.join( + device_dim, + "device_id", + "left" +).select( + iot_stream["*"], + device_dim["device_type"], + device_dim["location"], + device_dim["status"] +) + +enriched.writeStream \ + .format("delta") \ + .option("checkpointLocation", "/checkpoints/enriched") \ + .start("/delta/enriched_events") +``` + +### Pattern 2: Multi-Table Enrichment + +Chain multiple dimension joins: + +```python +# Multiple dimension tables +devices = spark.table("device_dimensions") +locations = spark.table("location_dimensions") +categories = spark.table("category_dimensions") + +# Chain joins (each is stateless) +enriched = (iot_stream + .join(devices, "device_id", "left") + .join(locations, "location_id", "left") + .join(categories, "category_id", "left") + .select( + iot_stream["*"], + devices["device_type"], + devices["manufacturer"], + locations["region"], + locations["country"], + categories["category_name"] + ) +) + +# Each join refreshes independently each microbatch +``` + +### Pattern 3: Broadcast Hash Join Optimization + +Optimize joins by ensuring broadcast: + +```python +from pyspark.sql.functions import broadcast + +# Option 1: Select only needed columns +small_dim = device_dim.select("device_id", "device_type", "location") + +# Option 2: Filter to active records +active_dim = device_dim.filter(col("status") == "active") + +# Option 3: Force broadcast hint +enriched = iot_stream.join( + broadcast(active_dim), + "device_id", + "left" +) + +# Verify in Spark UI: Look for "BroadcastHashJoin" in query plan +``` + +### Pattern 4: Audit Dimension Freshness + +Track how fresh dimension data is: + +```python +from pyspark.sql.functions import unix_timestamp, current_timestamp + +enriched = (iot_stream + .join(device_dim, "device_id", "left") + .withColumn( + "dim_lag_seconds", + unix_timestamp(current_timestamp()) - + unix_timestamp(col("dim_updated_at")) + ) + .withColumn( + "dim_fresh", + col("dim_lag_seconds") < 3600 # Less than 1 hour old + ) +) + +# Monitor: Alert if dim_lag_seconds > threshold +# Use for data quality checks +``` + +### Pattern 5: Time-Travel Dimension Lookup + +Join with dimension as-of event time: + +```python +from delta import DeltaTable + +def enrich_with_time_travel(batch_df, batch_id): + """Enrich with dimension version at event time""" + from pyspark.sql.functions import max as spark_max + + # Get latest dimension version + latest_version = DeltaTable.forName(spark, "device_dimensions") \ + .history() \ + .select(spark_max("version").alias("max_version")) \ + .first()[0] + + # Read dimension at specific version + dim_at_version = (spark + .read + .format("delta") + .option("versionAsOf", latest_version) + .table("device_dimensions") + ) + + # Join with batch + enriched = batch_df.join(dim_at_version, "device_id", "left") + + # Write + (enriched + .write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "enrichment_job") + .saveAsTable("enriched_events") + ) + +iot_stream.writeStream \ + .foreachBatch(enrich_with_time_travel) \ + .option("checkpointLocation", "/checkpoints/enriched") \ + .start() +``` + +### Pattern 6: Backfill Missing Dimensions + +Daily job to fix null dimensions from left join: + +```python +# Daily batch job to backfill missing dimensions +spark.sql(""" + MERGE INTO enriched_events target + USING device_dimensions source + ON target.device_id = source.device_id + AND target.device_type IS NULL + WHEN MATCHED THEN + UPDATE SET + device_type = source.device_type, + location = source.location, + manufacturer = source.manufacturer, + dim_updated_at = source.updated_at +""") + +# Run after dimension table updates +# Fixes events that arrived before dimension was available +``` + +### Pattern 7: Dimension Change Detection + +Stream that reacts to dimension changes: + +```python +def update_reference_cache(batch_df, batch_id): + """Update in-memory cache when dimension changes""" + # Dimension table changed + # Update application cache or notify downstream systems + pass + +# Stream dimension table changes +dim_changes = (spark + .readStream + .format("delta") + .table("device_dimensions") + .writeStream + .foreachBatch(update_reference_cache) + .option("checkpointLocation", "/checkpoints/dim_changes") + .start() +) +``` + +## Performance Optimization + +### Checklist + +- [ ] Dimension table < 100MB for broadcast (or increase threshold) +- [ ] Select only needed columns before join +- [ ] Filter dimension to active records only +- [ ] Verify "BroadcastHashJoin" in query plan +- [ ] Partition size 100-200MB in memory +- [ ] Use same region for compute and storage + +### Configuration + +```python +# Increase broadcast threshold if dimension is larger +spark.conf.set("spark.sql.autoBroadcastJoinThreshold", "1g") + +# Control partition size +spark.conf.set("spark.sql.shuffle.partitions", "200") + +# Optimize dimension table reads +spark.conf.set("spark.databricks.delta.optimizeWrite.enabled", "true") +spark.conf.set("spark.databricks.delta.autoCompact.enabled", "true") +``` + +### Reduce Dimension Size + +```python +# Before join: Select only needed columns +small_dim = device_dim.select( + "device_id", + "device_type", + "location", + "status" +) + +# Filter to active records +active_dim = small_dim.filter(col("status") == "active") + +# Join with smaller dimension +enriched = iot_stream.join(active_dim, "device_id", "left") +``` + +## Monitoring + +### Key Metrics + +```python +# Null rate (left join quality) +spark.sql(""" + SELECT + date_trunc('hour', timestamp) as hour, + count(*) as total_events, + count(device_type) as matched_events, + count(*) - count(device_type) as unmatched_events, + (count(*) - count(device_type)) * 100.0 / count(*) as null_rate_pct + FROM enriched_events + GROUP BY 1 + ORDER BY 1 DESC +""") + +# Dimension freshness +spark.sql(""" + SELECT + date_trunc('hour', timestamp) as hour, + avg(dim_lag_seconds) as avg_lag_seconds, + max(dim_lag_seconds) as max_lag_seconds, + count(*) as events_with_dim + FROM enriched_events + WHERE dim_updated_at IS NOT NULL + GROUP BY 1 + ORDER BY 1 DESC +""") +``` + +### Programmatic Monitoring + +```python +# Monitor stream health +for stream in spark.streams.active: + status = stream.status + progress = stream.lastProgress + + if progress: + print(f"Stream: {stream.name}") + print(f"Input rate: {progress.get('inputRowsPerSecond', 0)} rows/sec") + print(f"Processing rate: {progress.get('processedRowsPerSecond', 0)} rows/sec") + print(f"Batch duration: {progress.get('durationMs', {}).get('triggerExecution', 0)} ms") +``` + +### Spark UI Checks + +- **Streaming Tab**: Input rate vs processing rate (processing must exceed input) +- **SQL Tab**: Look for "BroadcastHashJoin" (not "SortMergeJoin") +- **Jobs Tab**: Check for shuffle operations (should be minimal) +- **Stages Tab**: Verify partition sizes (100-200MB target) + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **Data loss** | Inner join dropping unmatched events | Switch to left join | +| **Slow joins** | Shuffle join instead of broadcast | Reduce dimension size; force broadcast | +| **Stale data** | Non-Delta format | Convert dimension table to Delta | +| **Memory issues** | Large dimension table | Filter before join; increase broadcast threshold | +| **Skewed joins** | Hot keys in dimension | Salt the join key or partition dimension table | +| **High null rate** | Dimension updates lagging | Monitor dimension freshness; backfill job | + +## Production Best Practices + +### Always Use Left Join + +```python +# WRONG: Inner join loses data +enriched = iot_stream.join(device_dim, "device_id", "inner") + +# CORRECT: Left join preserves all events +enriched = iot_stream.join(device_dim, "device_id", "left") + +# Why? New devices may send data before dimension table is updated +# Left join preserves events; backfill dimensions later +``` + +### Handle Null Dimensions + +```python +# Add null handling in transformations +enriched = (iot_stream + .join(device_dim, "device_id", "left") + .withColumn( + "device_type", + coalesce(col("device_type"), lit("UNKNOWN")) + ) + .withColumn( + "location", + coalesce(col("location"), lit("UNKNOWN")) + ) +) + +# Or flag for manual review +enriched = enriched.withColumn( + "needs_review", + col("device_type").isNull() +) +``` + +### Idempotent Writes + +```python +def idempotent_write(batch_df, batch_id): + """Write with transaction version for idempotency""" + (batch_df + .write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "enrichment_job") + .saveAsTable("enriched_events") + ) + +enriched.writeStream \ + .foreachBatch(idempotent_write) \ + .option("checkpointLocation", "/checkpoints/enriched") \ + .start() +``` + +## Production Checklist + +- [ ] Left join used (not inner join) +- [ ] Dimension table is Delta format +- [ ] Broadcast hash join verified in query plan +- [ ] Dimension size optimized (< 100MB or threshold increased) +- [ ] Null rate monitored and alerts configured +- [ ] Dimension freshness tracked +- [ ] Backfill job scheduled for missing dimensions +- [ ] Checkpoint location is unique per query +- [ ] Idempotent writes configured (txnVersion/txnAppId) +- [ ] Performance metrics tracked (input rate, batch duration) + +## Expert Tips + +### Delta Version Checking + +Delta tables automatically refresh each microbatch by checking the latest version: + +```python +# Each microbatch: +# 1. Spark checks Delta table version +# 2. Reads latest version if changed +# 3. Uses cached version if unchanged +# 4. No manual refresh needed + +# This is why Delta tables work better than Parquet for dimensions +# Parquet: Read once at startup (truly static) +# Delta: Version checked each microbatch (semi-static) +``` + +### Broadcast Join Verification + +Always verify broadcast joins in production: + +```python +# Check query plan +enriched.explain(extended=True) + +# Look for: +# - BroadcastHashJoin ✅ (fast, no shuffle) +# - SortMergeJoin ⚠️ (slower, requires shuffle) + +# If seeing SortMergeJoin: +# 1. Reduce dimension size (select columns, filter rows) +# 2. Increase broadcast threshold +# 3. Force broadcast hint +``` + +### Dimension Table Optimization + +Optimize dimension tables for streaming joins: + +```python +# 1. Use Z-order or liquid clustering on join key +spark.sql(""" + OPTIMIZE device_dimensions + ZORDER BY (device_id) +""") + +# 2. Keep dimension tables small (< 100MB ideal) +# 3. Use Delta for automatic version checking +# 4. Partition by frequently filtered columns +``` + +## Related Skills + +- `stream-stream-joins` - Join two streaming sources with state management +- `kafka-to-delta` - Kafka ingestion patterns +- `write-multiple-tables` - Fan-out patterns for multiple sinks +- `checkpoint-best-practices` - Checkpoint configuration and management diff --git a/.claude/skills/databricks-spark-structured-streaming/stream-stream-joins.md b/.claude/skills/databricks-spark-structured-streaming/stream-stream-joins.md new file mode 100644 index 00000000..e5b10aad --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/stream-stream-joins.md @@ -0,0 +1,588 @@ +--- +name: stream-stream-joins +description: Join two streaming sources in real-time with event-time semantics, watermarks, and state management. Use when correlating events from different streams (orders with payments, clicks with conversions, sensor readings), handling late-arriving data, or implementing windowed aggregations across multiple streams. +--- + +# Stream-Stream Joins + +Join two streaming sources in real-time to correlate events that arrive at different times and speeds. Stream-stream joins require watermarks to manage state and handle late-arriving data. + +## Quick Start + +```python +from pyspark.sql.functions import expr, from_json, col +from pyspark.sql.types import StructType + +# Read two streaming sources +orders = (spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", "broker:9092") + .option("subscribe", "orders") + .load() + .select(from_json(col("value").cast("string"), order_schema).alias("data")) + .select("data.*") + .withWatermark("order_time", "10 minutes") +) + +payments = (spark + .readStream + .format("kafka") + .option("kafka.bootstrap.servers", "broker:9092") + .option("subscribe", "payments") + .load() + .select(from_json(col("value").cast("string"), payment_schema).alias("data")) + .select("data.*") + .withWatermark("payment_time", "10 minutes") +) + +# Join with time bounds +matched = (orders + .join( + payments, + expr(""" + orders.order_id = payments.order_id AND + payments.payment_time >= orders.order_time - interval 5 minutes AND + payments.payment_time <= orders.order_time + interval 10 minutes + """), + "inner" + ) +) + +# Write results +query = (matched + .writeStream + .format("delta") + .outputMode("append") + .option("checkpointLocation", "/Volumes/catalog/checkpoints/orders_payments") + .trigger(processingTime="30 seconds") + .start("/delta/order_payments") +) +``` + +## Core Concepts + +### Why Stream-Stream Joins Need Watermarks + +Stream-stream joins are stateful: both sides must buffer events until matches are found or state expires. Watermarks define when state can be safely cleaned up. + +```python +# Watermark = latest_event_time - delay_threshold +.withWatermark("event_time", "10 minutes") + +# Events with timestamp < watermark are considered "too late" +# State for late events is automatically cleaned up +``` + +### Join Types and Behavior + +| Join Type | Matches | Late Events | Use Case | +|-----------|---------|-------------|----------| +| **Inner** | Both sides | May still match if other side hasn't expired | Correlation analysis | +| **Left Outer** | All left + matched right | Dropped from left side after watermark | Enrichment with optional data | +| **Right Outer** | All right + matched left | Dropped from right side after watermark | Rarely used | +| **Full Outer** | All events from both | Dropped after watermark | Complete picture | + +## Common Patterns + +### Pattern 1: Order-Payment Matching + +Match orders with payments within a time window: + +```python +orders = (spark + .readStream + .format("kafka") + .option("subscribe", "orders") + .load() + .select(from_json(col("value").cast("string"), order_schema).alias("data")) + .select("data.*") + .withWatermark("order_time", "10 minutes") +) + +payments = (spark + .readStream + .format("kafka") + .option("subscribe", "payments") + .load() + .select(from_json(col("value").cast("string"), payment_schema).alias("data")) + .select("data.*") + .withWatermark("payment_time", "10 minutes") +) + +# Match payments within 10 minutes of order +matched = (orders + .join( + payments, + expr(""" + orders.order_id = payments.order_id AND + payments.payment_time >= orders.order_time - interval 5 minutes AND + payments.payment_time <= orders.order_time + interval 10 minutes + """), + "leftOuter" # Include orders without payments + ) + .withColumn("matched", col("payment_id").isNotNull()) +) + +matched.writeStream \ + .format("delta") \ + .option("checkpointLocation", "/checkpoints/orders_payments") \ + .start("/delta/order_payments") +``` + +### Pattern 2: Click-Conversion Attribution + +Attribute conversions to clicks within a time window: + +```python +impressions = (spark + .readStream + .format("kafka") + .option("subscribe", "impressions") + .load() + .select(from_json(col("value").cast("string"), impression_schema).alias("data")) + .select("data.*") + .withWatermark("impression_time", "1 hour") +) + +conversions = (spark + .readStream + .format("kafka") + .option("subscribe", "conversions") + .load() + .select(from_json(col("value").cast("string"), conversion_schema).alias("data")) + .select("data.*") + .withWatermark("conversion_time", "1 hour") +) + +# Attribute conversion to last impression within 24 hours +attributed = (impressions + .join( + conversions, + expr(""" + impressions.user_id = conversions.user_id AND + impressions.ad_id = conversions.ad_id AND + conversions.conversion_time >= impressions.impression_time AND + conversions.conversion_time <= impressions.impression_time + interval 24 hours + """), + "inner" + ) + .withColumn("attribution_window_hours", + (col("conversion_time").cast("long") - col("impression_time").cast("long")) / 3600) +) + +attributed.writeStream \ + .format("delta") \ + .option("checkpointLocation", "/checkpoints/attribution") \ + .start("/delta/attributed_conversions") +``` + +### Pattern 3: Sessionization Across Streams + +Group events from multiple streams into sessions: + +```python +from pyspark.sql.functions import session_window + +pageviews = (spark + .readStream + .format("kafka") + .option("subscribe", "pageviews") + .load() + .select(from_json(col("value").cast("string"), pageview_schema).alias("data")) + .select("data.*") + .withWatermark("event_time", "30 minutes") +) + +clicks = (spark + .readStream + .format("kafka") + .option("subscribe", "clicks") + .load() + .select(from_json(col("value").cast("string"), click_schema).alias("data")) + .select("data.*") + .withWatermark("event_time", "30 minutes") +) + +# Create session windows for each stream +pageview_sessions = (pageviews + .groupBy( + col("user_id"), + session_window(col("event_time"), "10 minutes") + ) + .agg( + count("*").alias("pageview_count"), + min("event_time").alias("session_start"), + max("event_time").alias("session_end") + ) +) + +click_sessions = (clicks + .groupBy( + col("user_id"), + session_window(col("event_time"), "10 minutes") + ) + .agg( + count("*").alias("click_count"), + min("event_time").alias("session_start"), + max("event_time").alias("session_end") + ) +) + +# Join sessions +joined_sessions = (pageview_sessions + .join( + click_sessions, + ["user_id", "session_window"], + "outer" + ) + .withColumn("total_events", + coalesce(col("pageview_count"), lit(0)) + + coalesce(col("click_count"), lit(0))) +) + +joined_sessions.writeStream \ + .format("delta") \ + .option("checkpointLocation", "/checkpoints/sessions") \ + .start("/delta/user_sessions") +``` + +### Pattern 4: Late Data Handling with Dead Letter Queue + +Route late-arriving events to a separate table: + +```python +def write_with_late_data_handling(batch_df, batch_id): + """Separate on-time and late data""" + from pyspark.sql.functions import current_timestamp, unix_timestamp + + # Calculate delay + processed = batch_df.withColumn( + "processing_delay_seconds", + unix_timestamp(current_timestamp()) - unix_timestamp(col("event_time")) + ) + + # On-time data (within watermark) + on_time = processed.filter(col("processing_delay_seconds") < 600) # 10 minutes + + # Late data + late = processed.filter(col("processing_delay_seconds") >= 600) + + # Write on-time data + (on_time + .drop("processing_delay_seconds") + .write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "stream_join_job") + .saveAsTable("matched_events") + ) + + # Write late data to DLQ + if late.count() > 0: + (late + .withColumn("dlq_reason", lit("LATE_ARRIVAL")) + .withColumn("dlq_timestamp", current_timestamp()) + .write + .format("delta") + .mode("append") + .saveAsTable("late_data_dlq") + ) + +matched.writeStream \ + .foreachBatch(write_with_late_data_handling) \ + .option("checkpointLocation", "/checkpoints/orders_payments") \ + .start() +``` + +## State Management + +### Configure RocksDB for Large State + +For state stores exceeding memory capacity, use RocksDB: + +```python +# Enable RocksDB state store provider +spark.conf.set( + "spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider" +) + +# State is stored on disk, reducing memory pressure +# Recommended for: High cardinality keys, long watermark durations +``` + +### Monitor State Size + +```python +# Read state store directly +state_df = (spark + .read + .format("statestore") + .load("/checkpoints/orders_payments/state") +) + +# Check partition balance +state_df.groupBy("partitionId").count().orderBy(desc("count")).show() + +# Check state size +state_metadata = (spark + .read + .format("state-metadata") + .load("/checkpoints/orders_payments") +) +state_metadata.show() + +# Programmatic monitoring +for stream in spark.streams.active: + progress = stream.lastProgress + if progress and "stateOperators" in progress: + for op in progress["stateOperators"]: + print(f"State rows: {op.get('numRowsTotal', 0)}") + print(f"State memory: {op.get('memoryUsedBytes', 0)}") +``` + +### Control State Growth + +```python +# 1. Use watermarks (automatic cleanup) +.withWatermark("event_time", "10 minutes") # State expires after watermark + +# 2. Reduce key cardinality +# Bad: user_id (millions of distinct values) +# Good: session_id (expires naturally) + +# 3. Set reasonable time bounds +# Bad: unbounded time range +expr("s2.ts >= s1.ts") # State grows forever! + +# Good: bounded time range +expr("s2.ts BETWEEN s1.ts AND s1.ts + interval 1 hour") +``` + +## Watermark Configuration + +### Choosing Watermark Duration + +Balance between latency and completeness: + +```python +# Rule of thumb: 2-3x the expected delay +# If 99th percentile delay is 5 minutes → use 10-15 minute watermark + +# High tolerance (more matches, larger state) +.withWatermark("event_time", "2 hours") + +# Low tolerance (faster results, smaller state) +.withWatermark("event_time", "10 minutes") +``` + +### Multiple Watermarks + +When joining streams with different latencies: + +```python +# Stream 1: Fast, low latency +stream1 = stream1.withWatermark("ts", "5 minutes") + +# Stream 2: Slow, high latency +stream2 = stream2.withWatermark("ts", "15 minutes") + +# Effective watermark = max(5, 15) = 15 minutes +joined = stream1.join(stream2, join_condition, "inner") +``` + +## Production Best Practices + +### Idempotent Writes + +Ensure exactly-once semantics: + +```python +def idempotent_write(batch_df, batch_id): + """Write with transaction version for idempotency""" + (batch_df + .write + .format("delta") + .mode("append") + .option("txnVersion", batch_id) + .option("txnAppId", "stream_join_job") + .saveAsTable("matched_events") + ) + +matched.writeStream \ + .foreachBatch(idempotent_write) \ + .option("checkpointLocation", "/checkpoints/orders_payments") \ + .start() +``` + +### Multi-Stream Joins (3+ Streams) + +Chain joins carefully - each adds state overhead: + +```python +# Step 1: Join streams A and B +ab = (stream_a + .withWatermark("ts", "10 minutes") + .join( + stream_b.withWatermark("ts", "10 minutes"), + expr("a.key = b.key AND b.ts BETWEEN a.ts - interval 5 min AND a.ts + interval 5 min"), + "inner" + ) +) + +# Step 2: Join result with stream C +abc = ab.join( + stream_c.withWatermark("ts", "10 minutes"), + expr("ab.key = c.key AND c.ts BETWEEN ab.ts - interval 5 min AND ab.ts + interval 5 min"), + "inner" +) + +# Note: Result watermark comes from left side (ab) +``` + +### Performance Tuning + +```python +# State store batch retention +spark.conf.set("spark.sql.streaming.stateStore.minBatchesToRetain", "2") + +# State maintenance interval +spark.conf.set("spark.sql.streaming.stateStore.maintenanceInterval", "5m") + +# Shuffle partitions (match worker cores) +spark.conf.set("spark.sql.shuffle.partitions", "200") +``` + +## Monitoring + +### Key Metrics + +```python +# Programmatic monitoring +for stream in spark.streams.active: + status = stream.status + progress = stream.lastProgress + + if progress: + print(f"Stream: {stream.name}") + print(f"Input rate: {progress.get('inputRowsPerSecond', 0)} rows/sec") + print(f"Processing rate: {progress.get('processedRowsPerSecond', 0)} rows/sec") + + # State metrics + if "stateOperators" in progress: + for op in progress["stateOperators"]: + print(f"State rows: {op.get('numRowsTotal', 0)}") + print(f"State memory: {op.get('memoryUsedBytes', 0)}") + + # Watermark + if "eventTime" in progress: + print(f"Watermark: {progress['eventTime'].get('watermark', 'N/A')}") +``` + +### Spark UI Checks + +- **Streaming Tab**: Input rate vs processing rate (processing must exceed input) +- **State Operators**: State size and memory usage +- **Watermark**: Current watermark timestamp +- **Batch Duration**: Should be < trigger interval + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **State too large** | High cardinality keys or long watermark | Reduce key space; decrease watermark duration | +| **Late events dropped** | Watermark too aggressive | Increase watermark delay | +| **No matches** | Time condition wrong | Check time bounds and units (minutes vs hours) | +| **OOM errors** | State explosion | Use RocksDB; increase memory; reduce watermark | +| **Missing watermarks** | State grows forever | Always define watermarks on both sides | +| **Unbounded state** | Open-ended time range | Use bounded time range in join condition | + +## Production Checklist + +- [ ] Watermark configured on both streaming sources +- [ ] Join condition includes explicit time bounds +- [ ] State store provider set (RocksDB for large state) +- [ ] State size monitored and alerts configured +- [ ] Late data handling strategy defined (DLQ or tolerance) +- [ ] Output mode is "append" (required for streaming joins) +- [ ] Checkpoint location is unique per query +- [ ] Idempotent writes configured (txnVersion/txnAppId) +- [ ] Time zones normalized across streams +- [ ] Performance metrics tracked (input rate, state size, watermark lag) + +## Expert Tips + +### Event Time vs Processing Time + +Always use event time for stream-stream joins: + +```python +# ✅ CORRECT: Event time (deterministic) +.withWatermark("event_time", "10 minutes") + +# ❌ WRONG: Processing time (non-deterministic) +# Processing time varies based on system load +# Results are not reproducible +``` + +### Watermark Semantics Deep Dive + +Understanding watermark behavior: + +```python +# Watermark = max_event_time - delay_threshold +# Example: max_event_time = 10:15, delay = 10 min +# Watermark = 10:05 + +# Events with timestamp < 10:05 are "too late" +# - Inner join: May still match if other side hasn't expired +# - Outer join: Dropped from outer side after watermark passes + +# Effective watermark = max(left_watermark, right_watermark) +``` + +### State Store Backend Selection + +Choose the right state store backend: + +```python +# Default: In-memory (fast but limited) +# Use for: Small state (< 10GB), low cardinality keys + +# RocksDB: Disk-backed (slower but scalable) +spark.conf.set( + "spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider" +) +# Use for: Large state (> 10GB), high cardinality keys + +# Monitor state size to decide when to switch +``` + +### Join Condition Best Practices + +Always include explicit time bounds: + +```python +# ❌ BAD: Unbounded (state grows forever) +expr("s1.key = s2.key AND s2.ts >= s1.ts") + +# ✅ GOOD: Bounded (state bounded by watermark) +expr(""" + s1.key = s2.key AND + s2.ts >= s1.ts - interval 5 minutes AND + s2.ts <= s1.ts + interval 10 minutes +""") + +# Why? Bounded ranges allow state cleanup +# Unbounded ranges cause state to grow indefinitely +``` + +## Related Skills + +- `stream-static-joins` - Enrich streams with Delta dimension tables +- `kafka-to-delta` - Kafka ingestion patterns +- `watermark-configuration` - Deep dive on watermark semantics +- `state-store-management` - State store optimization and monitoring diff --git a/.claude/skills/databricks-spark-structured-streaming/streaming-best-practices.md b/.claude/skills/databricks-spark-structured-streaming/streaming-best-practices.md new file mode 100644 index 00000000..9f3927a9 --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/streaming-best-practices.md @@ -0,0 +1,265 @@ +--- +name: "streaming-best-practices" +description: "Production-proven best practices for Spark Streaming: trigger intervals, partitioning, checkpoint management, and cluster configuration for reliable pipelines." +tags: ["spark-streaming", "best-practices", "production", "performance", "expert"] +--- + +# Streaming Best Practices Expert Pack + +## Overview + +A comprehensive checklist distilled from production experience. These practices should hold true in almost all scenarios. + +**Source**: Canadian Data Guy — "Spark Streaming Best Practices" + +## Beginner Checklist + +### 1. Always Set a Trigger Interval + +```python +# ✅ Good: Controls API costs and listing operations +stream.writeStream \ + .trigger(processingTime='5 seconds') \ + .start() + +# ❌ Bad: No trigger means continuous microbatches +# Can cause excessive S3/ADLS listing costs +``` + +**Why**: Fast processing (<1 sec) repeats listing operations, causing unintended costs. + +### 2. Use Auto Loader Notification Mode + +```python +# Switch from file listing to event-based +spark.readStream \ + .format("cloudFiles") \ + .option("cloudFiles.useNotifications", "true") \ + .load("/path/to/data") +``` + +[Auto Loader File Notification Mode](https://docs.databricks.com/ingestion/auto-loader/file-notification-mode.html) + +### 3. Disable S3 Versioning + +```python +# ❌ Don't enable versioning on S3 buckets with Delta +# ✅ Delta has time travel — no need for S3 versioning +# Versioning adds significant latency at scale +``` + +### 4. Co-Locate Compute and Storage + +```python +# ✅ Keep compute and storage in the same region +# Cross-region = latency + egress costs +``` + +### 5. Use ADLS Gen2 on Azure + +```python +# ✅ ADLS Gen2 is optimized for big data analytics +# ❌ Regular blob storage = slower performance +``` + +### 6. Partition Strategy + +```python +# ✅ Partition on low-cardinality columns: date, region, country +# ❌ Avoid high-cardinality: user_id, transaction_id + +# Rule of thumb: < 100,000 partitions +# Example: 10 years × 365 days × 20 countries = 73,000 partitions ✅ +``` + +### 7. Name Your Streaming Query + +```python +# ✅ Easily identifiable in Spark UI +stream.writeStream \ + .option("queryName", "IngestFromKafka") \ + .start() + +# Shows up as "IngestFromKafka" in Streaming tab +``` + +### 8. One Checkpoint Per Stream + +```python +# ✅ Each stream has its own checkpoint +# ❌ Never share checkpoints between streams + +# Example: Two sources → one target +# Source 1 → checkpoint_1 → target +# Source 2 → checkpoint_2 → target +``` + +### 9. Don't Multiplex Streams + +```python +# ❌ Don't run multiple streams on same driver +# Can cause stability issues + +# ✅ Use separate jobs or benchmark thoroughly +``` + +### 10. Optimal Partition Size + +```python +# Target: 100-200MB per partition in memory + +# Tune with: +.option("maxFilesPerTrigger", "100") +.option("maxBytesPerTrigger", "100MB") + +# Monitor in Spark UI → Stages → Partition size +``` + +### 11. Prefer Broadcast Hash Join + +```python +# ✅ BroadcastHashJoin is faster than SortMergeJoin +# Spark auto-broadcasts tables < 100MB + +# Increase threshold if needed: +spark.conf.set("spark.sql.autoBroadcastJoinThreshold", "1g") +``` + +## Advanced Checklist + +### 12. Checkpoint Naming Convention + +```python +# Structure: {table_location}/_checkpoints/_{target_table_name}_starting_{identifier} + +# Examples: +# 1. By timestamp: /delta/events/_checkpoints/_events_starting_2024_01_15 +# 2. By version: /delta/events/_checkpoints/_events_startingVersion_12345 + +# Why: Multiple checkpoints over table lifetime (upgrades, logic changes) +``` + +### 13. Minimize Shuffle Spill + +```python +# ✅ Goal: Shuffle spill (disk) = 0 +# ✅ Only shuffle read should exist + +# Check: Spark UI → SQL → Exchange operators +# If spill > 0: Increase memory or reduce partition size +``` + +### 14. Use RocksDB for Stateful Operations + +```python +# For large state stores, use RocksDB backend +spark.conf.set( + "spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider" +) +``` + +### 15. Event Hubs via Kafka Connector + +```python +# ✅ Use Kafka protocol for Azure Event Hubs +# More flexible partition handling + +# Note: With EventHubs Kafka connector +# Number of cores can differ from partitions +# (vs native EventHubs: cores == partitions) +``` + +### 16. Watermark for State Cleanup + +```python +# ✅ Always use watermark with stateful ops +# Prevents infinite state growth + +stream.withWatermark("timestamp", "10 minutes") \ + .groupBy("user_id") \ + .agg(sum("amount")) + +# Exception: If infinite state needed, store in Delta + ZORDER +``` + +### 17. Deduplication at Scale + +```python +# At trillion-record scale: +# ✅ Delta merge over dropDuplicates + +# dropDuplicates: State store grows very large +# Delta merge: Use table for lookup + +# Example: +spark.sql(""" + MERGE INTO target t + USING source s ON t.event_id = s.event_id + WHEN NOT MATCHED THEN INSERT * +""") +``` + +### 18. Azure Instance Family Selection + +| Workload | Instance Family | +|----------|----------------| +| Map-heavy (parsing, JSON) | F-series | +| Multiple streams from same source | Fsv2-series | +| Joins/aggregations/optimize | DS_v2-series | +| Delta caching | L-series (SSD) | + +### 19. Shuffle Partitions + +```python +# Set equal to total worker cores +spark.conf.set("spark.sql.shuffle.partitions", "200") + +# ❌ Don't set too high +# If changing: Clear checkpoint (stores the old value) +``` + +## Quick Reference + +### Trigger Selection + +| Latency Requirement | Trigger | +|---------------------|---------| +| < 1 second | Real-Time Mode (RTM) | +| 1-10 seconds | processingTime('5 seconds') | +| 1-60 minutes | processingTime based on SLA/3 | +| Batch-like | availableNow=True | + +### Cluster Sizing + +```python +# Fixed-size cluster recommended for streaming +# ❌ Don't use auto-scaling for streaming workloads + +# Why: Pre-allocated resources = predictable latency +``` + +## Monitoring Checklist + +- [ ] Input rate vs processing rate (processing > input) +- [ ] Max offsets behind latest (should decrease over time) +- [ ] Batch duration vs trigger interval (headroom exists) +- [ ] State store size (if using stateful ops) +- [ ] Shuffle spill = 0 +- [ ] Null rate in left joins (data quality) + +## Common Mistakes + +| Mistake | Impact | Fix | +|---------|--------|-----| +| Shared checkpoint | Data loss/corruption | Separate checkpoints | +| No watermark | State explosion | Add watermark | +| S3 versioning | Latency | Disable versioning | +| Autoscaling clusters | Unpredictable latency | Fixed-size clusters | +| High-cardinality partitions | Small files | Partition by date | + +## Related Skills + +- `spark-streaming-master-class-kafka-to-delta` — End-to-end patterns +- `mastering-checkpoints-in-spark-streaming` — Checkpoint deep dive +- `scaling-spark-streaming-jobs` — Performance tuning diff --git a/.claude/skills/databricks-spark-structured-streaming/trigger-and-cost-optimization.md b/.claude/skills/databricks-spark-structured-streaming/trigger-and-cost-optimization.md new file mode 100644 index 00000000..92ba4cb3 --- /dev/null +++ b/.claude/skills/databricks-spark-structured-streaming/trigger-and-cost-optimization.md @@ -0,0 +1,517 @@ +--- +name: trigger-and-cost-optimization +description: Select and tune triggers for Spark Structured Streaming to balance latency and cost. Use when choosing between processingTime, availableNow, and Real-Time Mode (RTM), calculating optimal trigger intervals, optimizing costs through cluster right-sizing, scheduled streaming, multi-stream clusters, or managing latency vs cost trade-offs. +--- + +# Trigger and Cost Optimization + +Select and tune triggers to balance latency requirements with cost. Optimize streaming job costs through trigger tuning, cluster right-sizing, multi-stream clusters, storage optimization, and scheduled execution patterns. + +## Quick Start + +```python +# Cost-optimized: Scheduled streaming instead of continuous +df.writeStream \ + .format("delta") \ + .option("checkpointLocation", "/checkpoints/stream") \ + .trigger(availableNow=True) \ # Process all, then stop + .start("/delta/target") + +# Schedule via Databricks Jobs: Every 15 minutes +# Cost: ~$20/day for 100 tables on 8-core cluster +``` + +## Trigger Types + +### ProcessingTime Trigger + +Process at fixed intervals: + +```python +# Process every 30 seconds +.trigger(processingTime="30 seconds") + +# Process every 5 minutes +.trigger(processingTime="5 minutes") + +# Latency: Trigger interval + processing time +# Cost: Continuous cluster running +``` + +### AvailableNow Trigger + +Process all available data, then stop: + +```python +# Process all available data, then stop +.trigger(availableNow=True) + +# Schedule via Databricks Jobs: +# - Every 15 minutes: Near real-time +# - Every 4 hours: Batch-style + +# Latency: Schedule interval + processing time +# Cost: Cluster runs only during processing +``` + +### Real-Time Mode (RTM) + +Sub-second latency with Photon: + +```python +# Real-Time Mode (Databricks 13.3+) +.trigger(realTime=True) + +# Requirements: +# - Photon enabled +# - Fixed-size cluster (no autoscaling) +# - Latency: < 800ms + +# Cost: Continuous cluster with Photon +``` + +## Trigger Selection Guide + +| Latency Requirement | Trigger | Cost | Use Case | +|---------------------|---------|------|----------| +| < 800ms | RTM | $$$ | Real-time analytics, alerts | +| 1-30 seconds | processingTime | $$ | Near real-time dashboards | +| 15-60 minutes | availableNow (scheduled) | $ | Batch-style SLA | +| > 1 hour | availableNow (scheduled) | $ | ETL pipelines | + +## Trigger Interval Calculation + +### Rule of Thumb: SLA / 3 + +```python +# Calculate trigger interval from SLA +business_sla_minutes = 60 # 1 hour SLA +trigger_interval_minutes = business_sla_minutes / 3 # 20 minutes + +.trigger(processingTime=f"{trigger_interval_minutes} minutes") + +# Why /3? +# - Processing time buffer +# - Recovery time buffer +# - Safety margin +``` + +### Example Calculations + +```python +# Example 1: 1 hour SLA +sla = 60 # minutes +trigger = sla / 3 # 20 minutes +.trigger(processingTime="20 minutes") + +# Example 2: 15 minute SLA +sla = 15 # minutes +trigger = sla / 3 # 5 minutes +.trigger(processingTime="5 minutes") + +# Example 3: Real-time requirement +.trigger(realTime=True) # < 800ms +``` + +## Cost Optimization Strategies + +### Strategy 1: Trigger Interval Tuning + +Balance latency and cost: + +```python +# Shorter interval = higher cost +.trigger(processingTime="5 seconds") # Expensive - continuous processing + +# Longer interval = lower cost +.trigger(processingTime="5 minutes") # Cheaper - less frequent processing + +# Use availableNow for batch-style (cheapest) +.trigger(availableNow=True) # Process backlog, then stop + +# Rule of thumb: SLA / 3 +# Example: 1 hour SLA → 20 minute trigger +``` + +### Strategy 2: Scheduled vs Continuous + +Choose execution pattern based on SLA: + +| Pattern | Cost | Latency | Use Case | +|---------|------|---------|----------| +| Continuous | $$$ | < 1 minute | Real-time requirements | +| 15-min schedule | $$ | 15-30 minutes | Near real-time | +| 4-hour schedule | $ | 4-5 hours | Batch-style SLA | + +```python +# Continuous (expensive) +.trigger(processingTime="30 seconds") + +# Scheduled (cost-effective) +.trigger(availableNow=True) # Schedule via Jobs: Every 15 minutes + +# Batch-style (cheapest) +.trigger(availableNow=True) # Schedule via Jobs: Every 4 hours +``` + +### Strategy 3: Cluster Right-Sizing + +Right-size clusters based on workload: + +```python +# Don't oversize: +# - Monitor CPU utilization (target 60-80%) +# - Check for idle time +# - Use fixed-size clusters (no autoscaling for streaming) + +# Scale test approach: +# 1. Start small +# 2. Monitor lag (max offsets behind latest) +# 3. Scale up if falling behind +# 4. Right-size based on steady state +``` + +### Strategy 4: Multi-Stream Clusters + +Run multiple streams on one cluster: + +```python +# Run multiple streams on one cluster +# Tested: 100 streams on 8-core single-node cluster +# Cost: ~$20/day for 100 tables + +# Example: Multiple streams on same cluster +stream1.writeStream.option("checkpointLocation", "/checkpoints/stream1").start() +stream2.writeStream.option("checkpointLocation", "/checkpoints/stream2").start() +stream3.writeStream.option("checkpointLocation", "/checkpoints/stream3").start() +# ... up to 100+ streams + +# Monitor: CPU/memory per stream +# Scale cluster if aggregate utilization > 80% +``` + +### Strategy 5: Storage Optimization + +Reduce storage costs: + +```sql +-- VACUUM old files +VACUUM table RETAIN 24 HOURS; + +-- Enable auto-optimize to reduce small files +ALTER TABLE table SET TBLPROPERTIES ( + 'delta.autoOptimize.optimizeWrite' = true, + 'delta.autoOptimize.autoCompact' = true +); + +-- Archive old data to cheaper storage +-- Use data retention policies +``` + +## Cost Formula + +``` +Daily Cost = + (Cluster DBU/hour × Hours running) + + (Storage GB × Storage rate) + + (Network egress if applicable) + +Optimization levers: +- Reduce hours running (scheduled triggers) +- Reduce cluster size (right-sizing) +- Reduce storage (VACUUM, compression) +- Reduce network egress (co-locate compute and storage) +``` + +## Common Patterns + +### Pattern 1: Cost-Optimized Scheduled Streaming + +Convert continuous to scheduled: + +```python +# Before: Continuous (expensive) +df.writeStream \ + .trigger(processingTime="30 seconds") \ + .start() + +# After: Scheduled (cost-effective) +df.writeStream \ + .trigger(availableNow=True) \ # Process all, then stop + .start() + +# Schedule via Databricks Jobs: +# - Every 15 minutes: Near real-time +# - Every 4 hours: Batch-style +# Same code, different schedule +``` + +### Pattern 2: Multi-Stream Cluster + +Optimize cluster utilization: + +```python +# Run multiple streams on one cluster +def start_all_streams(): + streams = [] + + # Start multiple streams + for i in range(100): + stream = (spark + .readStream + .table(f"source_{i}") + .writeStream + .format("delta") + .option("checkpointLocation", f"/checkpoints/stream_{i}") + .trigger(availableNow=True) + .start(f"/delta/target_{i}") + ) + streams.append(stream) + + return streams + +# Monitor aggregate CPU/memory +# Scale cluster if needed +``` + +### Pattern 3: RTM for Sub-Second Latency + +Use RTM for real-time requirements: + +```python +# Real-Time Mode for sub-second latency +df.writeStream \ + .format("kafka") + .option("topic", "output") + .trigger(realTime=True) \ + .start() + +# Required configurations: +spark.conf.set("spark.databricks.photon.enabled", "true") +spark.conf.set("spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider") + +# Latency: < 800ms +# Cost: Continuous cluster with Photon +``` + +## Real-Time Mode (RTM) Configuration + +### Enable RTM + +```python +# Enable Real-Time Mode +.trigger(realTime=True) + +# Required configurations: +spark.conf.set("spark.databricks.photon.enabled", "true") +spark.conf.set("spark.sql.streaming.stateStore.providerClass", + "com.databricks.sql.streaming.state.RocksDBStateProvider") + +# Cluster requirements: +# - Fixed-size cluster (no autoscaling) +# - Photon enabled +# - Driver: Minimum 4 cores +``` + +### RTM Use Cases + +```python +# Good for RTM: +# - Sub-second latency requirements +# - Simple transformations +# - Stateless operations +# - Kafka-to-Kafka pipelines + +# Not recommended for RTM: +# - Stateful operations (aggregations, joins) +# - Complex transformations +# - Large batch sizes +``` + +## Performance Considerations + +### Batch Duration vs Trigger Interval + +```python +# Batch duration should be < trigger interval +# Example: +trigger_interval = 30 # seconds +batch_duration = 10 # seconds + +# Healthy: batch_duration < trigger_interval +# Unhealthy: batch_duration >= trigger_interval + +# Monitor in Spark UI: +# - Batch duration +# - Trigger interval +# - Alert if batch duration >= trigger interval +``` + +### Trigger Interval Tuning + +```python +# Start conservative, optimize based on monitoring +# Step 1: Start with SLA / 3 +trigger_interval = business_sla / 3 + +# Step 2: Monitor batch duration +# If batch duration < trigger_interval / 2: Can increase trigger +# If batch duration >= trigger_interval: Decrease trigger + +# Step 3: Optimize for cost vs latency +# Increase trigger interval to reduce cost +# Decrease trigger interval to reduce latency +``` + +## Cost Monitoring + +### Track Per-Stream Costs + +```python +# Tag jobs with stream name +job_tags = { + "stream_name": "orders_stream", + "environment": "prod", + "cost_center": "analytics" +} + +# Use DBU consumption metrics +# Monitor by workspace/cluster +# Track cost per stream over time +``` + +### Monitor Cluster Utilization + +```python +# Check CPU utilization +# Target: 60-80% utilization +# Below 60%: Consider downsizing +# Above 80%: Consider upsizing + +# Check memory utilization +# Monitor for OOM errors +# Adjust cluster size accordingly +``` + +## Latency vs Cost Trade-offs + +### Continuous Processing + +```python +# High cost, low latency +.trigger(processingTime="30 seconds") + +# Cost: Continuous cluster running +# Latency: 30 seconds + processing time +# Use when: Real-time requirements +``` + +### Scheduled Processing + +```python +# Lower cost, higher latency +.trigger(availableNow=True) # Schedule: Every 15 minutes + +# Cost: Cluster runs only during processing +# Latency: Schedule interval + processing time +# Use when: Batch-style SLA acceptable +``` + +### Real-Time Mode + +```python +# Highest cost, lowest latency +.trigger(realTime=True) + +# Cost: Continuous cluster with Photon +# Latency: < 800ms +# Use when: Sub-second latency required +``` + +## Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| **High latency** | Trigger interval too long | Decrease trigger interval or use RTM | +| **High cost** | Continuous processing | Use scheduled (availableNow) | +| **Batch duration > trigger** | Processing too slow | Optimize processing or increase trigger | +| **RTM not working** | Photon not enabled | Enable Photon and configure cluster | + +## Quick Wins + +1. **Change from continuous to 15-minute schedule** - Significant cost reduction +2. **Run multiple streams per cluster** - Better cluster utilization +3. **Enable auto-optimize** - Reduce storage costs +4. **Use Spot instances** - For non-critical streams (with caution) +5. **Archive old data** - Move to cheaper storage tiers + +## Trade-offs + +| Cost Reduction | Impact | Mitigation | +|----------------|--------|------------| +| Longer trigger | Higher latency | Acceptable if SLA allows | +| Smaller cluster | May fall behind | Monitor lag; scale if needed | +| Aggressive VACUUM | Less time travel | Balance retention vs cost | +| Spot instances | Possible interruptions | Use for non-critical streams | +| Scheduled vs continuous | Higher latency | Match to business SLA | + +## Production Best Practices + +### Match Trigger to SLA + +```python +# Calculate trigger from business SLA +def calculate_trigger_interval(sla_minutes): + """Calculate optimal trigger interval""" + return max(30, sla_minutes / 3) # Minimum 30 seconds + +trigger_interval = calculate_trigger_interval(business_sla_minutes) +.trigger(processingTime=f"{trigger_interval} seconds") +``` + +### Cluster Configuration + +```python +# Fixed-size cluster (no autoscaling for streaming) +cluster_config = { + "num_workers": 4, + "node_type_id": "i3.xlarge", + "autotermination_minutes": 60, # Terminate if idle + "enable_elastic_disk": True # Reduce storage costs +} +``` + +### Storage Management + +```sql +-- Enable auto-optimize +ALTER TABLE table SET TBLPROPERTIES ( + 'delta.autoOptimize.optimizeWrite' = true, + 'delta.autoOptimize.autoCompact' = true +); + +-- Periodic VACUUM +VACUUM table RETAIN 7 DAYS; -- Balance retention vs cost + +-- Archive old partitions +-- Move to cheaper storage tier +``` + +## Production Checklist + +- [ ] Trigger type selected based on latency requirements +- [ ] Trigger interval calculated from SLA (SLA / 3) +- [ ] Batch duration monitored (< trigger interval) +- [ ] Cluster right-sized (60-80% utilization) +- [ ] Multiple streams per cluster (if applicable) +- [ ] Scheduled execution (if SLA allows) +- [ ] RTM configured if sub-second latency required +- [ ] Auto-optimize enabled +- [ ] Storage costs monitored +- [ ] Cost per stream tracked + +## Related Skills + +- `kafka-streaming` - RTM configuration for Kafka pipelines +- `checkpoint-best-practices` - Checkpoint management diff --git a/.claude/skills/synthetic-data-generation/SKILL.md b/.claude/skills/databricks-synthetic-data-generation/SKILL.md similarity index 98% rename from .claude/skills/synthetic-data-generation/SKILL.md rename to .claude/skills/databricks-synthetic-data-generation/SKILL.md index 6d029941..ce2a17cf 100644 --- a/.claude/skills/synthetic-data-generation/SKILL.md +++ b/.claude/skills/databricks-synthetic-data-generation/SKILL.md @@ -1,5 +1,5 @@ --- -name: synthetic-data-generation +name: databricks-synthetic-data-generation description: "Generate realistic synthetic data using Faker and Spark, with non-linear distributions, integrity constraints, and save to Databricks. Use when creating test data, demo datasets, or synthetic tables." --- @@ -652,3 +652,9 @@ This returns schema, row counts, and column statistics to confirm the data was w 14. **Always use files**: Write to local file, execute, edit if error, re-execute 15. **Context reuse**: Pass `cluster_id` and `context_id` for faster iterations 16. **Libraries**: Install `faker` and `holidays` first; most others are pre-installed + +## Related Skills + +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - for building bronze/silver/gold pipelines on top of generated data +- **[databricks-aibi-dashboards](../databricks-aibi-dashboards/SKILL.md)** - for visualizing the generated data in dashboards +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - for managing catalogs, schemas, and volumes where data is stored diff --git a/.claude/skills/databricks-unity-catalog/SKILL.md b/.claude/skills/databricks-unity-catalog/SKILL.md index b8dbbc20..9b77fed9 100644 --- a/.claude/skills/databricks-unity-catalog/SKILL.md +++ b/.claude/skills/databricks-unity-catalog/SKILL.md @@ -104,6 +104,13 @@ mcp__databricks__execute_sql( 3. **Grant minimal access** - System tables contain sensitive metadata 4. **Schedule reports** - Create scheduled queries for regular monitoring +## Related Skills + +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - for pipelines that write to Unity Catalog tables +- **[databricks-jobs](../databricks-jobs/SKILL.md)** - for job execution data visible in system tables +- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - for generating data stored in Unity Catalog Volumes +- **[databricks-aibi-dashboards](../databricks-aibi-dashboards/SKILL.md)** - for building dashboards on top of Unity Catalog data + ## Resources - [Unity Catalog System Tables](https://docs.databricks.com/administration-guide/system-tables/) diff --git a/.claude/skills/unstructured-pdf-generation/SKILL.md b/.claude/skills/databricks-unstructured-pdf-generation/SKILL.md similarity index 91% rename from .claude/skills/unstructured-pdf-generation/SKILL.md rename to .claude/skills/databricks-unstructured-pdf-generation/SKILL.md index 1c5a5a4f..7666f21b 100644 --- a/.claude/skills/unstructured-pdf-generation/SKILL.md +++ b/.claude/skills/databricks-unstructured-pdf-generation/SKILL.md @@ -1,5 +1,5 @@ --- -name: unstructured-pdf-generation +name: databricks-unstructured-pdf-generation description: "Generate synthetic PDF documents for RAG and unstructured data use cases. Use when creating test PDFs, demo documents, or evaluation datasets for retrieval systems." --- @@ -185,3 +185,10 @@ AZURE_OPENAI_DEPLOYMENT=gpt-4o | **"Volume does not exist"** | The tool creates volumes automatically; ensure you have CREATE VOLUME permission | | **"PDF generation timeout"** | Reduce `count` or check LLM endpoint availability | | **Low quality content** | Provide more detailed `description` with specific topics and document types | + +## Related Skills + +- **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Create Knowledge Assistants that ingest the generated PDFs +- **[databricks-vector-search](../databricks-vector-search/SKILL.md)** - Index generated documents for semantic search and RAG +- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate structured tabular data (complement to unstructured PDFs) +- **[databricks-mlflow-evaluation](../databricks-mlflow-evaluation/SKILL.md)** - Evaluate RAG systems using the generated question/guideline pairs diff --git a/.claude/skills/databricks-vector-search/SKILL.md b/.claude/skills/databricks-vector-search/SKILL.md new file mode 100644 index 00000000..276cab37 --- /dev/null +++ b/.claude/skills/databricks-vector-search/SKILL.md @@ -0,0 +1,357 @@ +--- +name: databricks-vector-search +description: "Patterns for Databricks Vector Search: create endpoints and indexes, query with filters, manage embeddings. Use when building RAG applications, semantic search, or similarity matching. Covers both storage-optimized and standard endpoints." +--- + +# Databricks Vector Search + +Patterns for creating, managing, and querying vector search indexes for RAG and semantic search applications. + +## When to Use + +Use this skill when: +- Building RAG (Retrieval-Augmented Generation) applications +- Implementing semantic search or similarity matching +- Creating vector indexes from Delta tables +- Choosing between storage-optimized and standard endpoints +- Querying vector indexes with filters + +## Overview + +Databricks Vector Search provides managed vector similarity search with automatic embedding generation and Delta Lake integration. + +| Component | Description | +|-----------|-------------| +| **Endpoint** | Compute resource hosting indexes (Standard or Storage-Optimized) | +| **Index** | Vector data structure for similarity search | +| **Delta Sync** | Auto-syncs with source Delta table | +| **Direct Access** | Manual CRUD operations on vectors | + +## Endpoint Types + +| Type | Latency | Capacity | Cost | Best For | +|------|---------|----------|------|----------| +| **Standard** | ~50-100ms | 320M vectors (768 dim) | Higher | Real-time, low-latency | +| **Storage-Optimized** | ~250ms | 1B+ vectors (768 dim) | 7x lower | Large-scale, cost-sensitive | + +## Index Types + +| Type | Embeddings | Sync | Use Case | +|------|------------|------|----------| +| **Delta Sync (managed)** | Databricks computes | Auto from Delta | Easiest setup | +| **Delta Sync (self-managed)** | You provide | Auto from Delta | Custom embeddings | +| **Direct Access** | You provide | Manual CRUD | Real-time updates | + +## Quick Start + +### Create Endpoint + +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +# Create a standard endpoint +endpoint = w.vector_search_endpoints.create_endpoint( + name="my-vs-endpoint", + endpoint_type="STANDARD" # or "STORAGE_OPTIMIZED" +) +# Note: Endpoint creation is asynchronous; check status with get_endpoint() +``` + +### Create Delta Sync Index (Managed Embeddings) + +```python +# Source table must have: primary key column + text column +index = w.vector_search_indexes.create_index( + name="catalog.schema.my_index", + endpoint_name="my-vs-endpoint", + primary_key="id", + index_type="DELTA_SYNC", + delta_sync_index_spec={ + "source_table": "catalog.schema.documents", + "embedding_source_columns": [ + { + "name": "content", # Text column to embed + "embedding_model_endpoint_name": "databricks-gte-large-en" + } + ], + "pipeline_type": "TRIGGERED" # or "CONTINUOUS" + } +) +``` + +### Query Index + +```python +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content", "metadata"], + query_text="What is machine learning?", + num_results=5 +) + +for doc in results.result.data_array: + score = doc[-1] # Similarity score is last column + print(f"Score: {score}, Content: {doc[1][:100]}...") +``` + +## Common Patterns + +### Create Storage-Optimized Endpoint + +```python +# For large-scale, cost-effective deployments +endpoint = w.vector_search_endpoints.create_endpoint( + name="my-storage-endpoint", + endpoint_type="STORAGE_OPTIMIZED" +) +``` + +### Delta Sync with Self-Managed Embeddings + +```python +# Source table must have: primary key + embedding vector column +index = w.vector_search_indexes.create_index( + name="catalog.schema.my_index", + endpoint_name="my-vs-endpoint", + primary_key="id", + index_type="DELTA_SYNC", + delta_sync_index_spec={ + "source_table": "catalog.schema.documents", + "embedding_vector_columns": [ + { + "name": "embedding", # Pre-computed embedding column + "embedding_dimension": 768 + } + ], + "pipeline_type": "TRIGGERED" + } +) +``` + +### Direct Access Index + +```python +import json + +# Create index for manual CRUD +index = w.vector_search_indexes.create_index( + name="catalog.schema.direct_index", + endpoint_name="my-vs-endpoint", + primary_key="id", + index_type="DIRECT_ACCESS", + direct_access_index_spec={ + "embedding_vector_columns": [ + {"name": "embedding", "embedding_dimension": 768} + ], + "schema_json": json.dumps({ + "id": "string", + "text": "string", + "embedding": "array", + "metadata": "string" + }) + } +) + +# Upsert data +w.vector_search_indexes.upsert_data_vector_index( + index_name="catalog.schema.direct_index", + inputs_json=json.dumps([ + {"id": "1", "text": "Hello", "embedding": [0.1, 0.2, ...], "metadata": "doc1"}, + {"id": "2", "text": "World", "embedding": [0.3, 0.4, ...], "metadata": "doc2"}, + ]) +) + +# Delete data +w.vector_search_indexes.delete_data_vector_index( + index_name="catalog.schema.direct_index", + primary_keys=["1", "2"] +) +``` + +### Query with Embedding Vector + +```python +# When you have pre-computed query embedding +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "text"], + query_vector=[0.1, 0.2, 0.3, ...], # Your 768-dim vector + num_results=10 +) +``` + +### Hybrid Search (Semantic + Keyword) + +```python +# Combines vector similarity with keyword matching +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_text="machine learning algorithms", + query_type="hybrid", # Enable hybrid search + num_results=10 +) +``` + +## Filtering + +### Standard Endpoint Filters (Dictionary) + +```python +# filters_json uses dictionary format +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_text="machine learning", + num_results=10, + filters_json='{"category": "ai", "status": ["active", "pending"]}' +) +``` + +### Storage-Optimized Filters (SQL-like) + +```python +# filter_string uses SQL-like syntax +results = w.vector_search_indexes.query_index( + index_name="catalog.schema.my_index", + columns=["id", "content"], + query_text="machine learning", + num_results=10, + filter_string="category = 'ai' AND status IN ('active', 'pending')" +) + +# More filter examples +filter_string="price > 100 AND price < 500" +filter_string="department LIKE 'eng%'" +filter_string="created_at >= '2024-01-01'" +``` + +### Trigger Index Sync + +```python +# For TRIGGERED pipeline type, manually sync +w.vector_search_indexes.sync_index( + index_name="catalog.schema.my_index" +) +``` + +### Scan All Index Entries + +```python +# Retrieve all vectors (for debugging/export) +scan_result = w.vector_search_indexes.scan_index( + index_name="catalog.schema.my_index", + num_results=100 +) +``` + +## Reference Files + +- [index-types.md](index-types.md) - Detailed comparison of index types and creation patterns + +## CLI Quick Reference + +```bash +# List endpoints +databricks vector-search endpoints list + +# Create endpoint +databricks vector-search endpoints create \ + --name my-endpoint \ + --endpoint-type STANDARD + +# List indexes on endpoint +databricks vector-search indexes list-indexes \ + --endpoint-name my-endpoint + +# Get index status +databricks vector-search indexes get-index \ + --index-name catalog.schema.my_index + +# Sync index (for TRIGGERED) +databricks vector-search indexes sync-index \ + --index-name catalog.schema.my_index + +# Delete index +databricks vector-search indexes delete-index \ + --index-name catalog.schema.my_index +``` + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Index sync slow** | Use Storage-Optimized endpoints (20x faster indexing) | +| **Query latency high** | Use Standard endpoint for <100ms latency | +| **filters_json not working** | Storage-Optimized uses `filter_string` (SQL syntax) | +| **Embedding dimension mismatch** | Ensure query and index dimensions match | +| **Index not updating** | Check pipeline_type; use sync_index() for TRIGGERED | +| **Out of capacity** | Upgrade to Storage-Optimized (1B+ vectors) | + +## Embedding Models + +Databricks provides built-in embedding models: + +| Model | Dimensions | Use Case | +|-------|------------|----------| +| `databricks-gte-large-en` | 1024 | English text, high quality | +| `databricks-bge-large-en` | 1024 | English text, general | + +```python +# Use with managed embeddings +embedding_source_columns=[ + { + "name": "content", + "embedding_model_endpoint_name": "databricks-gte-large-en" + } +] +``` + +## MCP Tools + +The following MCP tools are available for managing Vector Search infrastructure. These are **management tools** for creating and configuring endpoints/indexes. For agent-runtime querying, use the Databricks managed Vector Search MCP server or `VectorSearchRetrieverTool`. + +### Endpoint Management + +| Tool | Description | +|------|-------------| +| `create_vs_endpoint` | Create a Vector Search endpoint (STANDARD or STORAGE_OPTIMIZED) | +| `get_vs_endpoint` | Get endpoint status and details | +| `list_vs_endpoints` | List all endpoints in the workspace | +| `delete_vs_endpoint` | Delete an endpoint (indexes must be deleted first) | + +### Index Management + +| Tool | Description | +|------|-------------| +| `create_vs_index` | Create a Delta Sync or Direct Access index | +| `get_vs_index` | Get index status and configuration | +| `list_vs_indexes` | List all indexes on an endpoint | +| `delete_vs_index` | Delete an index | +| `sync_vs_index` | Trigger sync for TRIGGERED pipeline indexes | + +### Query and Data + +| Tool | Description | +|------|-------------| +| `query_vs_index` | Query index with text, vector, or hybrid search (for testing) | +| `upsert_vs_data` | Upsert vectors into a Direct Access index | +| `delete_vs_data` | Delete vectors from a Direct Access index | +| `scan_vs_index` | Scan/export index entries (for debugging) | + +## Notes + +- **Storage-Optimized is newer** - Better for most use cases unless you need <100ms latency +- **Delta Sync recommended** - Easier than Direct Access for most scenarios +- **Hybrid search** - Available for both Delta Sync and Direct Access indexes +- **Management vs runtime** - MCP tools above handle lifecycle management; for agent tool-calling at runtime, use the Databricks managed Vector Search MCP server + +## Related Skills + +- **[databricks-model-serving](../databricks-model-serving/SKILL.md)** - Deploy agents that use VectorSearchRetrieverTool +- **[databricks-agent-bricks](../databricks-agent-bricks/SKILL.md)** - Knowledge Assistants use RAG over indexed documents +- **[databricks-unstructured-pdf-generation](../databricks-unstructured-pdf-generation/SKILL.md)** - Generate documents to index in Vector Search +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Manage the catalogs and tables that back Delta Sync indexes +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Build Delta tables used as Vector Search sources diff --git a/.claude/skills/databricks-vector-search/index-types.md b/.claude/skills/databricks-vector-search/index-types.md new file mode 100644 index 00000000..ebfc1c7e --- /dev/null +++ b/.claude/skills/databricks-vector-search/index-types.md @@ -0,0 +1,254 @@ +# Vector Search Index Types + +## Comparison Matrix + +| Feature | Delta Sync (Managed) | Delta Sync (Self-Managed) | Direct Access | +|---------|---------------------|---------------------------|---------------| +| **Embeddings** | Databricks computes | You provide | You provide | +| **Sync** | Auto from Delta | Auto from Delta | Manual CRUD | +| **Setup** | Easiest | Medium | Most control | +| **Source** | Delta table + text | Delta table + vectors | API calls | +| **Best for** | Quick start, RAG | Custom models | Real-time apps | + +## Delta Sync with Managed Embeddings + +Databricks automatically computes embeddings from your text column. + +### Requirements + +- Source Delta table with: + - Primary key column (unique identifier) + - Text column (content to embed) +- Embedding model endpoint (or use built-in) + +### Create Index + +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() + +index = w.vector_search_indexes.create_index( + name="catalog.schema.docs_index", + endpoint_name="my-vs-endpoint", + primary_key="doc_id", + index_type="DELTA_SYNC", + delta_sync_index_spec={ + "source_table": "catalog.schema.documents", + "embedding_source_columns": [ + { + "name": "content", + "embedding_model_endpoint_name": "databricks-gte-large-en" + } + ], + "pipeline_type": "TRIGGERED", # or "CONTINUOUS" + "columns_to_sync": ["doc_id", "content", "title", "category"] + } +) +``` + +### Pipeline Types + +| Type | Behavior | Cost | Use Case | +|------|----------|------|----------| +| `TRIGGERED` | Manual sync via API | Lower | Batch updates | +| `CONTINUOUS` | Auto-sync on changes | Higher | Real-time sync | + +### Source Table Example + +```sql +CREATE TABLE catalog.schema.documents ( + doc_id STRING, + title STRING, + content STRING, -- Text to embed + category STRING, + created_at TIMESTAMP +); +``` + +## Delta Sync with Self-Managed Embeddings + +You pre-compute embeddings and store them in the source table. + +### Requirements + +- Source Delta table with: + - Primary key column + - Embedding vector column (array of floats) + +### Create Index + +```python +index = w.vector_search_indexes.create_index( + name="catalog.schema.custom_index", + endpoint_name="my-vs-endpoint", + primary_key="id", + index_type="DELTA_SYNC", + delta_sync_index_spec={ + "source_table": "catalog.schema.embedded_docs", + "embedding_vector_columns": [ + { + "name": "embedding", + "embedding_dimension": 768 + } + ], + "pipeline_type": "TRIGGERED" + } +) +``` + +### Compute Embeddings + +```python +from databricks.sdk import WorkspaceClient +import pandas as pd + +w = WorkspaceClient() + +def get_embeddings(texts: list[str]) -> list[list[float]]: + """Call embedding endpoint for texts.""" + response = w.serving_endpoints.query( + name="databricks-gte-large-en", + input=texts + ) + return [item.embedding for item in response.data] + +# Add embeddings to your data +df = spark.table("catalog.schema.documents").toPandas() +df["embedding"] = get_embeddings(df["content"].tolist()) + +# Write back to Delta +spark.createDataFrame(df).write.mode("overwrite").saveAsTable( + "catalog.schema.embedded_docs" +) +``` + +### Source Table Example + +```sql +CREATE TABLE catalog.schema.embedded_docs ( + id STRING, + content STRING, + embedding ARRAY, -- Pre-computed embedding + metadata STRING +); +``` + +## Direct Access Index + +Full control over vector data via CRUD API. No Delta table sync. + +### Requirements + +- Define schema upfront +- Manage upsert/delete operations yourself + +### Create Index + +```python +import json + +index = w.vector_search_indexes.create_index( + name="catalog.schema.realtime_index", + endpoint_name="my-vs-endpoint", + primary_key="id", + index_type="DIRECT_ACCESS", + direct_access_index_spec={ + "embedding_vector_columns": [ + {"name": "embedding", "embedding_dimension": 768} + ], + "schema_json": json.dumps({ + "id": "string", + "text": "string", + "embedding": "array", + "category": "string", + "score": "float" + }) + } +) +``` + +### Upsert Data + +```python +import json + +# Insert or update vectors +w.vector_search_indexes.upsert_data_vector_index( + index_name="catalog.schema.realtime_index", + inputs_json=json.dumps([ + { + "id": "doc-001", + "text": "Machine learning basics", + "embedding": [0.1, 0.2, 0.3, ...], # 768 floats + "category": "ml", + "score": 0.95 + }, + { + "id": "doc-002", + "text": "Deep learning overview", + "embedding": [0.4, 0.5, 0.6, ...], + "category": "dl", + "score": 0.88 + } + ]) +) +``` + +### Delete Data + +```python +w.vector_search_indexes.delete_data_vector_index( + index_name="catalog.schema.realtime_index", + primary_keys=["doc-001", "doc-002"] +) +``` + +### Attach Embedding Model (Optional) + +For Direct Access with text queries: + +```python +# Create index with embedding model for query-time embedding +index = w.vector_search_indexes.create_index( + name="catalog.schema.hybrid_index", + endpoint_name="my-vs-endpoint", + primary_key="id", + index_type="DIRECT_ACCESS", + direct_access_index_spec={ + "embedding_vector_columns": [ + {"name": "embedding", "embedding_dimension": 768} + ], + "embedding_model_endpoint_name": "databricks-gte-large-en", # For query_text + "schema_json": json.dumps({...}) + } +) +``` + +## Choosing the Right Type + +``` +Start here: +│ +├─ Do you have pre-computed embeddings? +│ ├─ Yes → Do you want auto-sync from Delta? +│ │ ├─ Yes → Delta Sync (Self-Managed) +│ │ └─ No → Direct Access +│ │ +│ └─ No → Delta Sync (Managed Embeddings) +│ +└─ Do you need real-time updates (<1 sec)? + ├─ Yes → Direct Access + └─ No → Delta Sync (any type) +``` + +## Endpoint Selection + +After choosing index type, choose endpoint: + +| Scenario | Endpoint Type | +|----------|---------------| +| Need <100ms latency | Standard | +| >100M vectors | Storage-Optimized | +| Cost-sensitive | Storage-Optimized | +| Default choice | Storage-Optimized | diff --git a/.claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md b/.claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md new file mode 100644 index 00000000..10b07f67 --- /dev/null +++ b/.claude/skills/databricks-zerobus-ingest/1-setup-and-authentication.md @@ -0,0 +1,199 @@ +# Setup and Authentication + +Complete setup guide for Zerobus Ingest: endpoint configuration, service principal creation, table preparation, SDK installation, and firewall requirements. + +--- + +## 1. Determine Your Server Endpoint + +The Zerobus server endpoint format depends on your cloud provider: + +| Cloud | Server Endpoint Format | Workspace URL Format | +|-------|------------------------|----------------------| +| **AWS** | `.zerobus..cloud.databricks.com` | `https://.cloud.databricks.com` | +| **Azure** | `.zerobus..azuredatabricks.net` | `https://.azuredatabricks.net` | + +**Example (AWS):** +``` +Server endpoint: 1234567890123456.zerobus.us-west-2.cloud.databricks.com +Workspace URL: https://dbc-a1b2c3d4-e5f6.cloud.databricks.com +``` + +**Finding your workspace ID:** Extract the numeric ID from your workspace URL or workspace settings page. It is the first segment of the server endpoint. + +--- + +## 2. Create the Target Table + +Zerobus does **not** create or alter tables. You must pre-create your target table as a **managed Delta table** in Unity Catalog: + +```sql +CREATE TABLE catalog.schema.my_events ( + event_id STRING, + device_name STRING, + temp INT, + humidity LONG, + event_time TIMESTAMP +); +``` + +**Constraints:** +- Must be a **managed** Delta table (no external storage) +- Table names limited to ASCII letters, digits, and underscores +- Maximum 2000 columns +- Table must be in a [supported region](#supported-regions) + +--- + +## 3. Create a Service Principal + +Zerobus authenticates via OAuth2 service principals (M2M). Create one via the Databricks UI or CLI: + +### Via UI +1. Go to **Settings > Identity and Access > Service principals** +2. Click **Add service principal** +3. Generate an OAuth secret: note the **client ID** and **client secret** + +### Via Databricks CLI +```bash +databricks service-principals create --display-name "zerobus-producer" +``` + +### Grant Table Permissions + +The service principal needs catalog, schema, and table access: + +```sql +-- Grant catalog access +GRANT USE CATALOG ON CATALOG my_catalog TO ``; + +-- Grant schema access +GRANT USE SCHEMA ON SCHEMA my_catalog.my_schema TO ``; + +-- Grant table write access +GRANT MODIFY, SELECT ON TABLE my_catalog.my_schema.my_events TO ``; +``` + +**Tip:** For broader access (e.g., writing to multiple tables in a schema), grant `MODIFY` and `SELECT` at the schema level instead. + +--- + +## 4. Install the SDK + +### Python (3.9+) + +```bash +pip install databricks-zerobus-ingest-sdk +``` + +Or with a virtual environment: +```bash +uv pip install databricks-zerobus-ingest-sdk +``` + +### Java (8+) + +Maven: +```xml + + com.databricks + zerobus-ingest-sdk + 0.1.0 + +``` + +Gradle: +```groovy +implementation 'com.databricks:zerobus-ingest-sdk:0.1.0' +``` + +### Go (1.21+) + +```bash +go get github.com/databricks/zerobus-sdk-go +``` + +### TypeScript / Node.js (16+) + +```bash +npm install @databricks/zerobus-ingest-sdk +``` + +### Rust (1.70+) + +```bash +cargo add databricks-zerobus-ingest-sdk +cargo add tokio --features macros,rt-multi-thread +``` + +--- + +## 5. Configure Environment Variables + +Store credentials as environment variables rather than hardcoding them: + +```bash +export ZEROBUS_SERVER_ENDPOINT="1234567890123456.zerobus.us-west-2.cloud.databricks.com" +export DATABRICKS_WORKSPACE_URL="https://dbc-a1b2c3d4-e5f6.cloud.databricks.com" +export ZEROBUS_TABLE_NAME="my_catalog.my_schema.my_events" +export DATABRICKS_CLIENT_ID="" +export DATABRICKS_CLIENT_SECRET="" +``` + +--- + +## 6. Firewall Allowlisting + +If your client application sits behind a firewall, you must allowlist the Zerobus IP addresses for your region before testing connectivity. Contact your Databricks representative or consult the [Zerobus documentation](https://docs.databricks.com/aws/en/ingestion/zerobus-overview) for the current IP ranges. + +--- + +## Supported Regions + +Workspace and target tables must reside in a supported region for your cloud provider. + +### AWS + +| Region Code | Location | +|-------------|----------| +| `us-east-1` | US East (N. Virginia) | +| `us-east-2` | US East (Ohio) | +| `us-west-2` | US West (Oregon) | +| `eu-central-1` | Europe (Frankfurt) | +| `eu-west-1` | Europe (Ireland) | +| `ap-southeast-1` | Asia Pacific (Singapore) | +| `ap-southeast-2` | Asia Pacific (Sydney) | +| `ap-northeast-1` | Asia Pacific (Tokyo) | +| `ca-central-1` | Canada (Central) | + +### Azure + +| Region Code | Location | +|-------------|----------| +| `canadacentral` | Canada Central | +| `westus` | West US | +| `eastus` | East US | +| `eastus2` | East US 2 | +| `centralus` | Central US | +| `northcentralus` | North Central US | +| `swedencentral` | Sweden Central | +| `westeurope` | West Europe | +| `northeurope` | North Europe | +| `australiaeast` | Australia East | +| `southeastasia` | Southeast Asia | + +--- + +## Verification Checklist + +Before writing your first record, confirm: + +``` +- [ ] Server endpoint matches your cloud provider and region +- [ ] Workspace URL is correct +- [ ] Target table exists as a managed Delta table +- [ ] Service principal has USE CATALOG, USE SCHEMA, MODIFY, SELECT grants +- [ ] SDK is installed for your target language +- [ ] Environment variables are set (or credentials are configured in code) +- [ ] Firewall allows outbound connections to the Zerobus endpoint (if applicable) +``` diff --git a/.claude/skills/databricks-zerobus-ingest/2-python-client.md b/.claude/skills/databricks-zerobus-ingest/2-python-client.md new file mode 100644 index 00000000..ac95cd4c --- /dev/null +++ b/.claude/skills/databricks-zerobus-ingest/2-python-client.md @@ -0,0 +1,323 @@ +# Python Client + +Python SDK patterns for Zerobus Ingest: synchronous and asynchronous APIs, JSON and Protobuf flows, and a reusable client class. + +--- + +## SDK Imports + +```python +# Synchronous API +from zerobus.sdk.sync import ZerobusSdk + +# Asynchronous API (equivalent capabilities) +from zerobus.sdk.asyncio import ZerobusSdk as AsyncZerobusSdk + +# Shared types (used by both sync and async) +from zerobus.sdk.shared import ( + RecordType, + IngestRecordResponse, + StreamConfigurationOptions, + TableProperties, +) +``` + +--- + + + +--- + +## Protobuf Ingestion + +You must always use Protobuf +For type-safe production workloads, use Protobuf. First generate and compile your `.proto` (see [4-protobuf-schema.md](4-protobuf-schema.md)), then: + +```python +import os +from zerobus.sdk.sync import ZerobusSdk +from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties + +# Import your compiled protobuf module +import record_pb2 + +server_endpoint = os.environ["ZEROBUS_SERVER_ENDPOINT"] +workspace_url = os.environ["DATABRICKS_WORKSPACE_URL"] +table_name = os.environ["ZEROBUS_TABLE_NAME"] +client_id = os.environ["DATABRICKS_CLIENT_ID"] +client_secret = os.environ["DATABRICKS_CLIENT_SECRET"] + +sdk = ZerobusSdk(server_endpoint, workspace_url) + +options = StreamConfigurationOptions(record_type=RecordType.PROTO) +table_props = TableProperties(table_name, record_pb2.AirQuality.DESCRIPTOR) + +stream = sdk.create_stream(client_id, client_secret, table_props, options) + +try: + for i in range(100): + record = record_pb2.AirQuality( + device_name=f"sensor-{i}", + temp=22, + humidity=55, + ) + ack = stream.ingest_record(record) + ack.wait_for_ack() +finally: + stream.close() +``` + +--- + +## ACK Callback (Asynchronous Acknowledgment) + +Instead of blocking on each ACK, register a callback for background durability confirmation: + +```python +from zerobus.sdk.shared import IngestRecordResponse, StreamConfigurationOptions, RecordType + +def on_ack(response: IngestRecordResponse) -> None: + print(f"Durable up to offset: {response.durability_ack_up_to_offset}") + +options = StreamConfigurationOptions( + record_type=RecordType.JSON, + ack_callback=on_ack, +) + +# Create stream with callback +stream = sdk.create_stream(client_id, client_secret, table_props, options) + +try: + for i in range(1000): + record = {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} + stream.ingest_record(record) # Non-blocking, ACKs arrive via callback + stream.flush() # Ensure all buffered records are sent +finally: + stream.close() +``` + +--- + +## Reusable Client Class + +A production-ready wrapper with retry logic, reconnection, and both JSON and Protobuf support: + +```python +import os +import time +import logging +from typing import Optional, Callable + +from zerobus.sdk.sync import ZerobusSdk +from zerobus.sdk.shared import ( + RecordType, + IngestRecordResponse, + StreamConfigurationOptions, + TableProperties, +) + +logger = logging.getLogger(__name__) + + +class ZerobusClient: + """Reusable Zerobus Ingest client with retry and reconnection.""" + + def __init__( + self, + server_endpoint: str, + workspace_url: str, + table_name: str, + client_id: str, + client_secret: str, + record_type: RecordType = RecordType.JSON, + ack_callback: Optional[Callable[[IngestRecordResponse], None]] = None, + proto_descriptor=None, + ): + self.server_endpoint = server_endpoint + self.workspace_url = workspace_url + self.table_name = table_name + self.client_id = client_id + self.client_secret = client_secret + self.record_type = record_type + self.ack_callback = ack_callback + self.proto_descriptor = proto_descriptor + + self.sdk = ZerobusSdk(self.server_endpoint, self.workspace_url) + self.stream = None + + def init_stream(self) -> None: + """Open a new stream to the target table.""" + options = StreamConfigurationOptions( + record_type=self.record_type, + ack_callback=self.ack_callback, + ) + if self.record_type == RecordType.PROTO and self.proto_descriptor: + table_props = TableProperties(self.table_name, self.proto_descriptor) + else: + table_props = TableProperties(self.table_name) + + self.stream = self.sdk.create_stream( + self.client_id, self.client_secret, table_props, options + ) + logger.info("Zerobus stream initialized for %s", self.table_name) + + def ingest(self, payload, max_retries: int = 3) -> bool: + """Ingest a single record (dict for JSON, protobuf message for PROTO). + + Returns True on success, False after exhausting retries. + """ + for attempt in range(max_retries): + try: + if self.stream is None: + self.init_stream() + ack = self.stream.ingest_record(payload) + ack.wait_for_ack() + return True + except Exception as e: + err = str(e).lower() + logger.warning( + "Ingest attempt %d/%d failed: %s", attempt + 1, max_retries, e + ) + if "closed" in err or "connection" in err: + self.close() + self.init_stream() + if attempt < max_retries - 1: + time.sleep(2**attempt) # Exponential backoff: 1s, 2s, 4s + return False + + def flush(self) -> None: + """Flush buffered writes.""" + if self.stream: + self.stream.flush() + + def close(self) -> None: + """Close the stream and release resources.""" + if self.stream: + self.stream.close() + self.stream = None + + def __enter__(self): + self.init_stream() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.flush() + self.close() + return False +``` + +### Using the Client Class + +```python +# JSON flow with context manager +with ZerobusClient( + server_endpoint=os.environ["ZEROBUS_SERVER_ENDPOINT"], + workspace_url=os.environ["DATABRICKS_WORKSPACE_URL"], + table_name=os.environ["ZEROBUS_TABLE_NAME"], + client_id=os.environ["DATABRICKS_CLIENT_ID"], + client_secret=os.environ["DATABRICKS_CLIENT_SECRET"], + record_type=RecordType.JSON, +) as client: + for i in range(100): + client.ingest({"device_name": f"sensor-{i}", "temp": 22, "humidity": 55}) + +# Protobuf flow +import record_pb2 + +with ZerobusClient( + server_endpoint=os.environ["ZEROBUS_SERVER_ENDPOINT"], + workspace_url=os.environ["DATABRICKS_WORKSPACE_URL"], + table_name=os.environ["ZEROBUS_TABLE_NAME"], + client_id=os.environ["DATABRICKS_CLIENT_ID"], + client_secret=os.environ["DATABRICKS_CLIENT_SECRET"], + record_type=RecordType.PROTO, + proto_descriptor=record_pb2.AirQuality.DESCRIPTOR, +) as client: + for i in range(100): + record = record_pb2.AirQuality(device_name=f"sensor-{i}", temp=22, humidity=55) + client.ingest(record) +``` + +--- + +## Async Python API + +The SDK provides an equivalent async API for use with `asyncio`: + +```python +import asyncio +from zerobus.sdk.asyncio import ZerobusSdk as AsyncZerobusSdk +from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties + + +async def ingest_async(): + sdk = AsyncZerobusSdk(server_endpoint, workspace_url) + options = StreamConfigurationOptions(record_type=RecordType.JSON) + table_props = TableProperties(table_name) + + stream = await sdk.create_stream(client_id, client_secret, table_props, options) + + try: + for i in range(100): + record = {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} + ack = await stream.ingest_record(record) + await ack.wait_for_ack() + finally: + await stream.close() + + +asyncio.run(ingest_async()) +``` + +**Tip:** The sync and async APIs have equivalent capabilities. Choose based on your application architecture (FastAPI/aiohttp -> async; scripts/batch jobs -> sync). + +--- + +## Batch Pattern + +For higher throughput, send records without blocking on each ACK and flush at the end: + +```python +with ZerobusClient( + server_endpoint=os.environ["ZEROBUS_SERVER_ENDPOINT"], + workspace_url=os.environ["DATABRICKS_WORKSPACE_URL"], + table_name=os.environ["ZEROBUS_TABLE_NAME"], + client_id=os.environ["DATABRICKS_CLIENT_ID"], + client_secret=os.environ["DATABRICKS_CLIENT_SECRET"], + record_type=RecordType.JSON, + ack_callback=lambda resp: None, # Discard individual ACKs +) as client: + for i in range(10_000): + record = {"device_name": f"sensor-{i}", "temp": 22, "humidity": 55} + client.stream.ingest_record(record) # Non-blocking + # flush() and close() called automatically by context manager +``` diff --git a/.claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md b/.claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md new file mode 100644 index 00000000..217398c8 --- /dev/null +++ b/.claude/skills/databricks-zerobus-ingest/3-multilanguage-clients.md @@ -0,0 +1,314 @@ +# Multi-Language Clients + +Zerobus Ingest SDK examples for Java, Go, TypeScript/Node.js, and Rust. All languages follow the same core pattern: **SDK init -> create stream -> ingest records -> ACK -> flush -> close**. + +--- + +## Java (8+) + +### Installation + +Maven: +```xml + + com.databricks + zerobus-ingest-sdk + 0.1.0 + +``` + +### Protobuf Flow (Recommended) + +Java uses Protobuf by default. Generate and compile your `.proto` first (see [4-protobuf-schema.md](4-protobuf-schema.md)). + +```java +import com.databricks.zerobus.*; +import com.example.proto.Record.AirQuality; + +public class ZerobusProducer { + public static void main(String[] args) throws Exception { + String serverEndpoint = System.getenv("ZEROBUS_SERVER_ENDPOINT"); + String workspaceUrl = System.getenv("DATABRICKS_WORKSPACE_URL"); + String tableName = System.getenv("ZEROBUS_TABLE_NAME"); + String clientId = System.getenv("DATABRICKS_CLIENT_ID"); + String clientSecret = System.getenv("DATABRICKS_CLIENT_SECRET"); + + ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + + TableProperties tableProperties = new TableProperties<>( + tableName, + AirQuality.getDefaultInstance() + ); + + ZerobusStream stream = sdk.createStream( + tableProperties, clientId, clientSecret + ).join(); + + try { + for (int i = 0; i < 100; i++) { + AirQuality record = AirQuality.newBuilder() + .setDeviceName("sensor-" + i) + .setTemp(22) + .setHumidity(55) + .build(); + stream.ingestRecord(record).join(); + } + } finally { + stream.close(); + } + } +} +``` + +### Proto Generation for Java + +```bash +java -jar zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ + --uc-endpoint "https://dbc-a1b2c3d4-e5f6.cloud.databricks.com" \ + --client-id "$DATABRICKS_CLIENT_ID" \ + --client-secret "$DATABRICKS_CLIENT_SECRET" \ + --table "catalog.schema.table_name" \ + --output "record.proto" + +# Compile to Java +protoc --java_out=src/main/java record.proto +``` + +--- + +## Go (1.21+) + +### Installation + +```bash +go get github.com/databricks/zerobus-sdk-go +``` + +### JSON Flow + +```go +package main + +import ( + "fmt" + "log" + "os" + + zerobus "github.com/databricks/zerobus-go-sdk/sdk" +) + +func main() { + serverEndpoint := os.Getenv("ZEROBUS_SERVER_ENDPOINT") + workspaceURL := os.Getenv("DATABRICKS_WORKSPACE_URL") + tableName := os.Getenv("ZEROBUS_TABLE_NAME") + clientID := os.Getenv("DATABRICKS_CLIENT_ID") + clientSecret := os.Getenv("DATABRICKS_CLIENT_SECRET") + + sdk, err := zerobus.NewZerobusSdk(serverEndpoint, workspaceURL) + if err != nil { + log.Fatal(err) + } + defer sdk.Free() + + options := zerobus.DefaultStreamConfigurationOptions() + options.RecordType = zerobus.RecordTypeJson + + stream, err := sdk.CreateStream( + zerobus.TableProperties{TableName: tableName}, + clientID, clientSecret, options, + ) + if err != nil { + log.Fatal(err) + } + defer stream.Close() + + for i := 0; i < 100; i++ { + record := fmt.Sprintf( + `{"device_name": "sensor-%d", "temp": 22, "humidity": 55}`, i, + ) + ack, err := stream.IngestRecord(record) + if err != nil { + log.Printf("Ingest failed for record %d: %v", i, err) + continue + } + ack.Await() + } + + stream.Flush() +} +``` + +### Protobuf Flow + +```go +options := zerobus.DefaultStreamConfigurationOptions() +options.RecordType = zerobus.RecordTypeProto + +// Load compiled proto descriptor +tableProps := zerobus.TableProperties{ + TableName: tableName, + DescriptorProto: descriptorBytes, // compiled .proto descriptor +} + +stream, err := sdk.CreateStream(tableProps, clientID, clientSecret, options) +// ... ingest protobuf-serialized bytes ... +``` + +--- + +## TypeScript / Node.js (16+) + +### Installation + +```bash +npm install @databricks/zerobus-ingest-sdk +``` + +### JSON Flow + +```typescript +import { ZerobusSdk, RecordType } from "@databricks/zerobus-ingest-sdk"; + +const serverEndpoint = process.env.ZEROBUS_SERVER_ENDPOINT!; +const workspaceUrl = process.env.DATABRICKS_WORKSPACE_URL!; +const tableName = process.env.ZEROBUS_TABLE_NAME!; +const clientId = process.env.DATABRICKS_CLIENT_ID!; +const clientSecret = process.env.DATABRICKS_CLIENT_SECRET!; + +const sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + +const stream = await sdk.createStream( + { tableName }, + clientId, + clientSecret, + { recordType: RecordType.Json } +); + +try { + for (let i = 0; i < 100; i++) { + const record = { device_name: `sensor-${i}`, temp: 22, humidity: 55 }; + await stream.ingestRecord(record); + } + await stream.flush(); +} finally { + await stream.close(); +} +``` + +### With Error Handling + +```typescript +import { ZerobusSdk, RecordType } from "@databricks/zerobus-ingest-sdk"; + +async function ingestWithRetry( + stream: any, + record: Record, + maxRetries = 3 +): Promise { + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + await stream.ingestRecord(record); + return true; + } catch (error) { + console.warn(`Attempt ${attempt + 1}/${maxRetries} failed:`, error); + if (attempt < maxRetries - 1) { + await new Promise((r) => setTimeout(r, 2 ** attempt * 1000)); + } + } + } + return false; +} +``` + +--- + +## Rust (1.70+) + +### Installation + +```bash +cargo add databricks-zerobus-ingest-sdk +cargo add tokio --features macros,rt-multi-thread +``` + +### JSON Flow + +```rust +use databricks_zerobus_ingest_sdk::{ + RecordType, StreamConfigurationOptions, TableProperties, ZerobusSdk, +}; +use std::env; +use std::error::Error; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let server_endpoint = env::var("ZEROBUS_SERVER_ENDPOINT")?; + let workspace_url = env::var("DATABRICKS_WORKSPACE_URL")?; + let table_name = env::var("ZEROBUS_TABLE_NAME")?; + let client_id = env::var("DATABRICKS_CLIENT_ID")?; + let client_secret = env::var("DATABRICKS_CLIENT_SECRET")?; + + let table_properties = TableProperties { + table_name, + descriptor_proto: None, + }; + + let options = StreamConfigurationOptions { + record_type: RecordType::Json, + ..Default::default() + }; + + let sdk = ZerobusSdk::new(server_endpoint, workspace_url)?; + let mut stream = sdk + .create_stream(table_properties, client_id, client_secret, Some(options)) + .await?; + + for i in 0..100 { + let record = format!( + r#"{{"device_name": "sensor-{}", "temp": 22, "humidity": 55}}"#, + i + ); + let ack = stream.ingest_record(record.into_bytes()).await?; + ack.await?; + } + + stream.close().await?; + Ok(()) +} +``` + +### Protobuf Flow + +```rust +let table_properties = TableProperties { + table_name: table_name.clone(), + descriptor_proto: Some(proto_descriptor_bytes), +}; + +let options = StreamConfigurationOptions { + record_type: RecordType::Proto, + ..Default::default() +}; + +let mut stream = sdk + .create_stream(table_properties, client_id, client_secret, Some(options)) + .await?; + +// Ingest serialized protobuf bytes +let record_bytes = my_proto_message.encode_to_vec(); +let ack = stream.ingest_record(record_bytes).await?; +ack.await?; +``` + +--- + +## Language Comparison + +| Feature | Python | Java | Go | TypeScript | Rust | +|---------|--------|------|----|------------|------| +| Min version | 3.9+ | 8+ | 1.21+ | Node 16+ | 1.70+ | +| Package | `databricks-zerobus-ingest-sdk` | `com.databricks:zerobus-ingest-sdk` | `github.com/databricks/zerobus-sdk-go` | `@databricks/zerobus-ingest-sdk` | `databricks-zerobus-ingest-sdk` | +| Default serialization | JSON | Protobuf | JSON | JSON | JSON | +| Async API | Yes (separate module) | CompletableFuture | Goroutines | Native async/await | Tokio async/await | +| ACK pattern | `ack.wait_for_ack()` or callback | `.join()` | `ack.Await()` | Implicit in `await` | `ack.await?` | +| Proto generation | `python -m zerobus.tools.generate_proto` | JAR CLI tool | External `protoc` | External `protoc` | External `protoc` | diff --git a/.claude/skills/databricks-zerobus-ingest/4-protobuf-schema.md b/.claude/skills/databricks-zerobus-ingest/4-protobuf-schema.md new file mode 100644 index 00000000..c8796faf --- /dev/null +++ b/.claude/skills/databricks-zerobus-ingest/4-protobuf-schema.md @@ -0,0 +1,191 @@ +# Protobuf Schema Generation + +Generate `.proto` schemas from Unity Catalog table definitions, compile language bindings, and understand Delta-to-Protobuf type mappings. + +--- + +## Why Protobuf? + +| Aspect | JSON | Protobuf | +|--------|------|----------| +| **Type safety** | None (runtime errors on mismatch) | Compile-time type checking | +| **Schema evolution** | Manual; easy to break silently | Forward-compatible by design | +| **Performance** | Text parsing overhead | Binary encoding, smaller payloads | +| **Recommended for** | Prototyping, simple schemas | Production, complex schemas | + +**Recommendation:** Use Protobuf for any production workload. Use JSON only for quick prototyping or when the schema is trivial. + +--- + +## Generate .proto from a UC Table + +### Python + +```bash +python -m zerobus.tools.generate_proto \ + --uc-endpoint "https://dbc-a1b2c3d4-e5f6.cloud.databricks.com" \ + --client-id "$DATABRICKS_CLIENT_ID" \ + --client-secret "$DATABRICKS_CLIENT_SECRET" \ + --table "catalog.schema.table_name" \ + --output record.proto +``` + +### Java + +```bash +java -jar zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ + --uc-endpoint "https://dbc-a1b2c3d4-e5f6.cloud.databricks.com" \ + --client-id "$DATABRICKS_CLIENT_ID" \ + --client-secret "$DATABRICKS_CLIENT_SECRET" \ + --table "catalog.schema.table_name" \ + --output record.proto +``` + +The generated `.proto` file will contain a message definition matching the table schema, for example: + +```protobuf +syntax = "proto3"; + +message AirQuality { + string device_name = 1; + int32 temp = 2; + int64 humidity = 3; +} +``` + +--- + +## Compile Language Bindings + +### Python + +```bash +pip install grpcio-tools + +python -m grpc_tools.protoc \ + -I. \ + --python_out=. \ + record.proto +``` + +This generates `record_pb2.py`. Import and use it: + +```python +import record_pb2 + +record = record_pb2.AirQuality( + device_name="sensor-1", + temp=22, + humidity=55, +) +``` + +### Java + +```bash +protoc --java_out=src/main/java record.proto +``` + +Generates Java classes under `src/main/java/`. Usage: + +```java +import com.example.proto.Record.AirQuality; + +AirQuality record = AirQuality.newBuilder() + .setDeviceName("sensor-1") + .setTemp(22) + .setHumidity(55) + .build(); +``` + +### Go + +```bash +protoc --go_out=. record.proto +``` + +### Rust + +Use `prost` in `build.rs`: + +```rust +// build.rs +fn main() { + prost_build::compile_protos(&["record.proto"], &["."]).unwrap(); +} +``` + +--- + +## Delta-to-Protobuf Type Mappings + +| Delta / Spark Type | Protobuf Type | Notes | +|--------------------|---------------|-------| +| `STRING` | `string` | | +| `INT` / `INTEGER` | `int32` | | +| `LONG` / `BIGINT` | `int64` | | +| `FLOAT` | `float` | | +| `DOUBLE` | `double` | | +| `BOOLEAN` | `bool` | | +| `BINARY` | `bytes` | | +| `ARRAY` | `repeated T` | Element type maps recursively | +| `MAP` | `map` | Key must be string or integer type | +| `STRUCT` | Nested `message` | Fields map recursively | +| `DATE` | `int32` | Epoch days (days since 1970-01-01) | +| `TIMESTAMP` | `int64` | Epoch microseconds | +| `DECIMAL(p,s)` | `bytes` or `string` | Check generated .proto for exact mapping | +| `VARIANT` | `string` | JSON-encoded string | + +**Important:** The Protobuf schema must match the Delta table schema exactly (1:1 field mapping). If the table schema changes, regenerate the `.proto` and recompile. + +--- + +## Maximum Schema Size + +- Maximum **2000 columns** per proto schema +- Maximum **10 MB** per individual message (10,485,760 bytes) + +--- + +## Schema Evolution Workflow + +When your table schema changes: + +1. Alter the table in Unity Catalog (add columns, etc.) +2. Regenerate the `.proto` file using the generation command +3. Recompile language bindings +4. Update your producer code to populate new fields +5. Redeploy + +**Note:** Zerobus does not support automatic schema evolution. You must manage this process explicitly. + +--- + +## Using the Descriptor in Code + +### Python + +```python +from zerobus.sdk.shared import TableProperties, RecordType +import record_pb2 + +# Pass the DESCRIPTOR from the compiled module +table_props = TableProperties( + "catalog.schema.table_name", + record_pb2.AirQuality.DESCRIPTOR, +) +``` + +### Java + +```java +// Pass a default instance to extract the descriptor +TableProperties tableProperties = new TableProperties<>( + "catalog.schema.table_name", + AirQuality.getDefaultInstance() +); +``` + +### Go / Rust + +Pass the raw descriptor bytes when constructing `TableProperties`. diff --git a/.claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md b/.claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md new file mode 100644 index 00000000..7b8cb2b8 --- /dev/null +++ b/.claude/skills/databricks-zerobus-ingest/5-operations-and-limits.md @@ -0,0 +1,251 @@ +# Operations and Limits + +ACK handling, retry and reconnection patterns, throughput limits, delivery semantics, and operational constraints for Zerobus Ingest. + +--- + +## Acknowledgment (ACK) Handling + +Every ingested record returns a durability acknowledgment. An ACK indicates that **all records up to that offset** have been durably written to the target Delta table. + +### Strategies + +| Strategy | When to Use | Trade-off | +|----------|-------------|-----------| +| **Sync block per record** | Low-volume, strict ordering | Simplest; lower throughput | +| **ACK callback** | High-volume producers | Higher throughput; more complex | +| **Periodic flush** | Batch-oriented workloads | Best throughput; eventual consistency | + +### Sync Block (Python) + +```python +ack = stream.ingest_record(record) +ack.wait_for_ack() # Blocks until durable +``` + +### ACK Callback (Python) + +```python +from zerobus.sdk.shared import IngestRecordResponse + +last_acked_offset = 0 + +def on_ack(response: IngestRecordResponse) -> None: + global last_acked_offset + last_acked_offset = response.durability_ack_up_to_offset + +options = StreamConfigurationOptions( + record_type=RecordType.JSON, + ack_callback=on_ack, +) +``` + +### Flush-Based + +```python +# Send many records without blocking +for record in batch: + stream.ingest_record(record) + +# Flush ensures all buffered records are sent +stream.flush() +``` + +--- + +## Retry and Reconnection + +Zerobus streams can close due to server maintenance, network issues, or zone failures. Implement retry with exponential backoff and stream reinitialization. + +### Pattern (Any Language) + +``` +1. Attempt ingest +2. On connection/closed error: + a. Close the current stream + b. Wait with exponential backoff (1s, 2s, 4s, ...) + c. Reinitialize the stream + d. Retry the record +3. After max retries, log failure and escalate +``` + +### Python Implementation + +```python +import time +import logging + +logger = logging.getLogger(__name__) + +def ingest_with_retry(stream_factory, record, max_retries=5): + """Ingest a record with retry and stream reinitialization. + + Args: + stream_factory: Callable that returns a new stream. + record: The record to ingest. + max_retries: Maximum retry attempts. + """ + stream = stream_factory() + + for attempt in range(max_retries): + try: + ack = stream.ingest_record(record) + ack.wait_for_ack() + return stream # Return the (possibly new) stream + except Exception as e: + err = str(e).lower() + logger.warning("Attempt %d/%d failed: %s", attempt + 1, max_retries, e) + + if "closed" in err or "connection" in err or "unavailable" in err: + try: + stream.close() + except Exception: + pass + backoff = min(2 ** attempt, 30) # Cap at 30s + time.sleep(backoff) + stream = stream_factory() + elif attempt < max_retries - 1: + time.sleep(2 ** attempt) + else: + raise + + return stream +``` + +### Key Points + +- **Always reinitialize the stream** on connection errors, not just retry the same stream +- **Cap backoff** at a reasonable maximum (e.g., 30 seconds) +- **Log failures** with enough context to diagnose (endpoint, table, error message) +- **Design for at-least-once**: your downstream consumers should handle duplicate records + +--- + +## Delivery Semantics + +Zerobus provides **at-least-once** delivery guarantees: + +- Records may be delivered more than once (e.g., after a retry where the original was actually persisted) +- There is **no exactly-once** semantics +- Design your target table and downstream consumers to handle duplicates (e.g., deduplication via `MERGE` or unique constraints) + +--- + +## Throughput Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| **Throughput per stream** | 100 MB/s | Based on 1 KB messages | +| **Rows per stream** | 15,000 rows/s | | +| **Max message size** | 10 MB (10,485,760 bytes) | Per individual record | +| **Max columns** | 2,000 | Per proto schema / table | + +### Scaling Beyond One Stream + +If you need higher throughput than a single stream provides: + +- Open **multiple streams** to the same table from different clients +- Zerobus supports **thousands of concurrent clients** writing to the same table +- Partition your data across streams by key (e.g., device ID, region) +- Contact Databricks for custom throughput requirements + +--- + +## Regional Availability + +Workspace and target tables must be in a supported region for your cloud provider. + +### AWS Supported Regions + +| Region | Code | +|--------|------| +| US East (N. Virginia) | `us-east-1` | +| US East (Ohio) | `us-east-2` | +| US West (Oregon) | `us-west-2` | +| Europe (Frankfurt) | `eu-central-1` | +| Europe (Ireland) | `eu-west-1` | +| Asia Pacific (Singapore) | `ap-southeast-1` | +| Asia Pacific (Sydney) | `ap-southeast-2` | +| Asia Pacific (Tokyo) | `ap-northeast-1` | +| Canada (Central) | `ca-central-1` | + +### Azure Supported Regions + +| Region | Code | +|--------|------| +| Canada Central | `canadacentral` | +| West US | `westus` | +| East US | `eastus` | +| East US 2 | `eastus2` | +| Central US | `centralus` | +| North Central US | `northcentralus` | +| Sweden Central | `swedencentral` | +| West Europe | `westeurope` | +| North Europe | `northeurope` | +| Australia East | `australiaeast` | +| Southeast Asia | `southeastasia` | + +**Performance note:** Optimal throughput requires the client application and Zerobus endpoint to be in the **same region**. + +--- + +## Durability and Availability + +- **Single-AZ only**: Zerobus runs in a single availability zone. The service may experience downtime if that zone is unavailable. +- **No geographic redundancy**: Plan for zone outages in your producer's retry logic. +- **Maintenance windows**: The server may close streams during maintenance. Your client should handle reconnection gracefully. + +--- + +## Target Table Constraints + +| Constraint | Details | +|------------|---------| +| **Table type** | Managed Delta tables only (no external storage) | +| **Table names** | ASCII letters, digits, underscores only | +| **Schema changes** | No auto-evolution; regenerate proto and redeploy | +| **Table creation** | Zerobus does not create tables; pre-create via SQL DDL | +| **Table recreation** | Cannot recreate an existing target table via Zerobus | + +--- + +## Supported Data Types + +| Delta Type | Protobuf Type | Conversion Notes | +|------------|---------------|------------------| +| STRING | string | Direct mapping | +| INT / INTEGER | int32 | Direct mapping | +| LONG / BIGINT | int64 | Direct mapping | +| FLOAT | float | Direct mapping | +| DOUBLE | double | Direct mapping | +| BOOLEAN | bool | Direct mapping | +| BINARY | bytes | Direct mapping | +| ARRAY\ | repeated T | Recursive mapping | +| MAP\ | map\ | Key must be string or integer | +| STRUCT | nested message | Recursive mapping | +| DATE | int32 | Epoch days since 1970-01-01 | +| TIMESTAMP | int64 | Epoch microseconds | +| VARIANT | string | JSON-encoded string | + +--- + +## Monitoring and Observability + +Zerobus does not currently expose built-in metrics dashboards. Monitor your producers with: + +- **Application-level logging**: Log ACK offsets, retry counts, and error rates +- **ACK callback tracking**: Track the last-acked offset to measure ingestion lag +- **Table row counts**: Periodically query the target table to verify data is arriving +- **Health checks**: Attempt a lightweight ingest (or stream creation) to verify connectivity + +```python +# Simple health check +def check_zerobus_health(sdk, client_id, client_secret, table_props, options): + try: + stream = sdk.create_stream(client_id, client_secret, table_props, options) + stream.close() + return True + except Exception as e: + logger.error("Zerobus health check failed: %s", e) + return False +``` diff --git a/.claude/skills/databricks-zerobus-ingest/SKILL.md b/.claude/skills/databricks-zerobus-ingest/SKILL.md new file mode 100644 index 00000000..d29dc00f --- /dev/null +++ b/.claude/skills/databricks-zerobus-ingest/SKILL.md @@ -0,0 +1,228 @@ +--- +name: databricks-zerobus-ingest +description: "Build Zerobus Ingest clients for near real-time data ingestion into Databricks Delta tables via gRPC. Use when creating producers that write directly to Unity Catalog tables without a message bus, working with the Zerobus Ingest SDK in Python/Java/Go/TypeScript/Rust, generating Protobuf schemas from UC tables, or implementing stream-based ingestion with ACK handling and retry logic." +--- + +# Zerobus Ingest + +Build clients that ingest data directly into Databricks Delta tables via the Zerobus gRPC API. + +**Status:** Public Preview (currently free; Databricks plans to introduce charges in the future) + +**Documentation:** +- [Zerobus Overview](https://docs.databricks.com/aws/en/ingestion/zerobus-overview) +- [Zerobus Ingest SDK](https://docs.databricks.com/aws/en/ingestion/zerobus-ingest) +- [Zerobus Limits](https://docs.databricks.com/aws/en/ingestion/zerobus-limits) + +--- + +## What Is Zerobus Ingest? + +Zerobus Ingest is a serverless connector that enables direct, record-by-record data ingestion into Delta tables via gRPC. It eliminates the need for message bus infrastructure (Kafka, Kinesis, Event Hub) for lakehouse-bound data. The service validates schemas, materializes data to target tables, and sends durability acknowledgments back to the client. + +**Core pattern:** SDK init -> create stream -> ingest records -> handle ACKs -> flush -> close + +--- + +## Quick Decision: What Are You Building? + +| Scenario | Language | Serialization | Reference | +|----------|----------|---------------|-----------| +| Quick prototype / test harness | Python | JSON | [2-python-client.md](2-python-client.md) | +| Production Python producer | Python | Protobuf | [2-python-client.md](2-python-client.md) + [4-protobuf-schema.md](4-protobuf-schema.md) | +| JVM microservice | Java | Protobuf | [3-multilanguage-clients.md](3-multilanguage-clients.md) | +| Go service | Go | JSON or Protobuf | [3-multilanguage-clients.md](3-multilanguage-clients.md) | +| Node.js / TypeScript app | TypeScript | JSON | [3-multilanguage-clients.md](3-multilanguage-clients.md) | +| High-performance system service | Rust | JSON or Protobuf | [3-multilanguage-clients.md](3-multilanguage-clients.md) | +| Schema generation from UC table | Any | Protobuf | [4-protobuf-schema.md](4-protobuf-schema.md) | +| Retry / reconnection logic | Any | Any | [5-operations-and-limits.md](5-operations-and-limits.md) | + +If not speficfied, default to python. + +--- + +## Common Libraries + +These libraries are essential for ZeroBus data ingestion: + +- **databricks-sdk>=0.85.0**: Databricks workspace client for authentication and metadata +- **databricks-zerobus-ingest-sdk>=0.2.0**: ZeroBus SDK for high-performance streaming ingestion +- **grpcio-tools** +These are typically NOT pre-installed on Databricks. Install them using `execute_databricks_command` tool: +- `code`: "%pip install databricks-sdk>=VERSION databricks-zerobus-ingest-sdk>=VERSION" + +Save the returned `cluster_id` and `context_id` for subsequent calls. + +Smart Installation Approach + +# Check protobuf version first, then install compatible +grpcio-tools +import google.protobuf +runtime_version = google.protobuf.__version__ +print(f"Runtime protobuf version: {runtime_version}") + +if runtime_version.startswith("5.26") or +runtime_version.startswith("5.29"): + %pip install grpcio-tools==1.62.0 +else: + %pip install grpcio-tools # Use latest for newer protobuf +versions +--- + +## Prerequisites + +You must never execute the skill without confirming the below objects are valid: + +1. **A Unity Catalog managed Delta table** to ingest into +2. **A service principal id and secret** with `MODIFY` and `SELECT` on the target table +3. **The Zerobus server endpoint** for your workspace region +4. **The Zerobus Ingest SDK** installed for your target language + +See [1-setup-and-authentication.md](1-setup-and-authentication.md) for complete setup instructions. + +--- + +## Minimal Python Example (JSON) + +```python +from zerobus.sdk.sync import ZerobusSdk +from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties + +sdk = ZerobusSdk(server_endpoint, workspace_url) +options = StreamConfigurationOptions(record_type=RecordType.JSON) +table_props = TableProperties(table_name) + +stream = sdk.create_stream(client_id, client_secret, table_props, options) +try: + record = {"device_name": "sensor-1", "temp": 22, "humidity": 55} + ack = stream.ingest_record(record) + ack.wait_for_ack() +finally: + stream.close() +``` + +--- + +## Detailed guides + +| Topic | File | When to Read | +|-------|------|--------------| +| Setup & Auth | [1-setup-and-authentication.md](1-setup-and-authentication.md) | Endpoint formats, service principals, SDK install | +| Python Client | [2-python-client.md](2-python-client.md) | Sync/async Python, JSON and Protobuf flows, reusable client class | +| Multi-Language | [3-multilanguage-clients.md](3-multilanguage-clients.md) | Java, Go, TypeScript, Rust SDK examples | +| Protobuf Schema | [4-protobuf-schema.md](4-protobuf-schema.md) | Generate .proto from UC table, compile, type mappings | +| Operations & Limits | [5-operations-and-limits.md](5-operations-and-limits.md) | ACK handling, retries, reconnection, throughput limits, constraints | + +--- + +You must always follow all the steps in the Workslfow + +## Workflow +0. **Display the plan of your execution** +1. **Determinate the type of client** +2. **Get schema** Always use 4-protobuf-schema.md. Execute using the `run_python_file_on_databricks` MCP tool +3. **Write Python code to a local file follow the instructions in the relevant guide to ingest with zerobus** in the project (e.g., `scripts/zerobus_ingest.py`). +4. **Execute on Databricks** using the `run_python_file_on_databricks` MCP tool +5. **If execution fails**: Edit the local file to fix the error, then re-execute +6. **Reuse the context** for follow-up executions by passing the returned `cluster_id` and `context_id` + +--- + +## Important +- Never install local packages +- Always validate MCP server requirement before execution + +--- + +### Context Reuse Pattern + +The first execution auto-selects a running cluster and creates an execution context. **Reuse this context for follow-up calls** - it's much faster (~1s vs ~15s) and shares variables/imports: + +**First execution** - use `run_python_file_on_databricks` tool: +- `file_path`: "scripts/zerobus_ingest.py" + +Returns: `{ success, output, error, cluster_id, context_id, ... }` + +Save `cluster_id` and `context_id` for follow-up calls. + +**If execution fails:** +1. Read the error from the result +2. Edit the local Python file to fix the issue +3. Re-execute with same context using `run_python_file_on_databricks` tool: + - `file_path`: "scripts/zerobus_ingest.py" + - `cluster_id`: "" + - `context_id`: "" + +**Follow-up executions** reuse the context (faster, shares state): +- `file_path`: "scripts/validate_ingestion.py" +- `cluster_id`: "" +- `context_id`: "" + +### Handling Failures + +When execution fails: +1. Read the error from the result +2. **Edit the local Python file** to fix the issue +3. Re-execute using the same `cluster_id` and `context_id` (faster, keeps installed libraries) +4. If the context is corrupted, omit `context_id` to create a fresh one + +--- + +### Installing Libraries + +Databricks provides Spark, pandas, numpy, and common data libraries by default. **Only install a library if you get an import error.** + +Use `execute_databricks_command` tool: +- `code`: "%pip install databricks-zerobus-ingest-sdk>=0.2.0" +- `cluster_id`: "" +- `context_id`: "" + +The library is immediately available in the same context. + +**Note:** Keeping the same `context_id` means installed libraries persist across calls. + +## 🚨 Critical Learning: Timestamp Format Fix + +**BREAKTHROUGH**: ZeroBus requires **timestamp fields as Unix integer timestamps**, NOT string timestamps. +The timestamp generation must use microseconds for Databricks. + +--- + +## Key Concepts + +- **gRPC + Protobuf**: Zerobus uses gRPC as its transport protocol. Any application that can communicate via gRPC and construct Protobuf messages can produce to Zerobus. +- **JSON or Protobuf serialization**: JSON for quick starts; Protobuf for type safety, forward compatibility, and performance. +- **At-least-once delivery**: The connector provides at-least-once guarantees. Design consumers to handle duplicates. +- **Durability ACKs**: Each ingested record returns an ACK confirming durable write. ACKs indicate all records up to that offset have been durably written. +- **No table management**: Zerobus does not create or alter tables. You must pre-create your target table and manage schema evolution yourself. +- **Single-AZ durability**: The service runs in a single availability zone. Plan for potential zone outages. + +--- + +## Common Issues + +| Issue | Solution | +|-------|----------| +| **Connection refused** | Verify server endpoint format matches your cloud (AWS vs Azure). Check firewall allowlists. | +| **Authentication failed** | Confirm service principal client_id/secret. Verify GRANT statements on the target table. | +| **Schema mismatch** | Ensure record fields match the target table schema exactly. Regenerate .proto if table changed. | +| **Stream closed unexpectedly** | Implement retry with exponential backoff and stream reinitialization. See [5-operations-and-limits.md](5-operations-and-limits.md). | +| **Throughput limits hit** | Max 100 MB/s and 15,000 rows/s per stream. Open multiple streams or contact Databricks. | +| **Region not supported** | Check supported regions in [5-operations-and-limits.md](5-operations-and-limits.md). | +| **Table not found** | Ensure table is a managed Delta table in a supported region with correct three-part name. | + +--- + +## Related Skills + +- **[databricks-python-sdk](../databricks-python-sdk/SKILL.md)** - General SDK patterns and WorkspaceClient for table/schema management +- **[databricks-spark-declarative-pipelines](../databricks-spark-declarative-pipelines/SKILL.md)** - Downstream pipeline processing of ingested data +- **[databricks-unity-catalog](../databricks-unity-catalog/SKILL.md)** - Managing catalogs, schemas, and tables that Zerobus writes to +- **[databricks-synthetic-data-generation](../databricks-synthetic-data-generation/SKILL.md)** - Generate test data to feed into Zerobus producers +- **[databricks-config](../databricks-config/SKILL.md)** - Profile and authentication setup + +## Resources + +- [Zerobus Overview](https://docs.databricks.com/aws/en/ingestion/zerobus-overview) +- [Zerobus Ingest SDK](https://docs.databricks.com/aws/en/ingestion/zerobus-ingest) +- [Zerobus Limits](https://docs.databricks.com/aws/en/ingestion/zerobus-limits) diff --git a/.claude/skills/mlflow-evaluation/SKILL.md b/.claude/skills/mlflow-evaluation/SKILL.md deleted file mode 100644 index 322f6aaa..00000000 --- a/.claude/skills/mlflow-evaluation/SKILL.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -name: mlflow-evaluation -description: "MLflow 3 GenAI evaluation for agent development. Use when (1) writing mlflow.genai.evaluate() code, (2) creating @scorer functions, (3) building evaluation datasets from traces, (4) using built-in scorers (Guidelines, Correctness, Safety, RetrievalGroundedness), (5) analyzing traces for latency/errors/architecture, (6) optimizing agent context/prompts/token usage, (7) debugging evaluation failures. Covers the full eval workflow: trace analysis -> dataset building -> scorer creation -> evaluation execution." ---- - -# MLflow 3 GenAI Evaluation - -## Before Writing Any Code - -1. **Read GOTCHAS.md** - 15+ common mistakes that cause failures -2. **Read CRITICAL-interfaces.md** - Exact API signatures and data schemas - -## End-to-End Workflows - -Follow these workflows based on your goal. Each step indicates which reference files to read. - -### Workflow 1: First-Time Evaluation Setup - -For users new to MLflow GenAI evaluation or setting up evaluation for a new agent. - -| Step | Action | Reference Files | -|------|--------|-----------------| -| 1 | Understand what to evaluate | `user-journeys.md` (Journey 0: Strategy) | -| 2 | Learn API patterns | `GOTCHAS.md` + `CRITICAL-interfaces.md` | -| 3 | Build initial dataset | `patterns-datasets.md` (Patterns 1-4) | -| 4 | Choose/create scorers | `patterns-scorers.md` + `CRITICAL-interfaces.md` (built-in list) | -| 5 | Run evaluation | `patterns-evaluation.md` (Patterns 1-3) | - -### Workflow 2: Production Trace -> Evaluation Dataset - -For building evaluation datasets from production traces. - -| Step | Action | Reference Files | -|------|--------|-----------------| -| 1 | Search and filter traces | `patterns-trace-analysis.md` (MCP tools section) | -| 2 | Analyze trace quality | `patterns-trace-analysis.md` (Patterns 1-7) | -| 3 | Tag traces for inclusion | `patterns-datasets.md` (Patterns 16-17) | -| 4 | Build dataset from traces | `patterns-datasets.md` (Patterns 6-7) | -| 5 | Add expectations/ground truth | `patterns-datasets.md` (Pattern 2) | - -### Workflow 3: Performance Optimization - -For debugging slow or expensive agent execution. - -| Step | Action | Reference Files | -|------|--------|-----------------| -| 1 | Profile latency by span | `patterns-trace-analysis.md` (Patterns 4-6) | -| 2 | Analyze token usage | `patterns-trace-analysis.md` (Pattern 9) | -| 3 | Detect context issues | `patterns-context-optimization.md` (Section 5) | -| 4 | Apply optimizations | `patterns-context-optimization.md` (Sections 1-4, 6) | -| 5 | Re-evaluate to measure impact | `patterns-evaluation.md` (Pattern 6-7) | - -### Workflow 4: Regression Detection - -For comparing agent versions and finding regressions. - -| Step | Action | Reference Files | -|------|--------|-----------------| -| 1 | Establish baseline | `patterns-evaluation.md` (Pattern 4: named runs) | -| 2 | Run current version | `patterns-evaluation.md` (Pattern 1) | -| 3 | Compare metrics | `patterns-evaluation.md` (Patterns 6-7) | -| 4 | Analyze failing traces | `patterns-trace-analysis.md` (Pattern 7) | -| 5 | Debug specific failures | `patterns-trace-analysis.md` (Patterns 8-9) | - -### Workflow 5: Custom Scorer Development - -For creating project-specific evaluation metrics. - -| Step | Action | Reference Files | -|------|--------|-----------------| -| 1 | Understand scorer interface | `CRITICAL-interfaces.md` (Scorer section) | -| 2 | Choose scorer pattern | `patterns-scorers.md` (Patterns 4-11) | -| 3 | For multi-agent scorers | `patterns-scorers.md` (Patterns 13-16) | -| 4 | Test with evaluation | `patterns-evaluation.md` (Pattern 1) | - -## Reference Files Quick Lookup - -| Reference | Purpose | When to Read | -|-----------|---------|--------------| -| `GOTCHAS.md` | Common mistakes | **Always read first** before writing code | -| `CRITICAL-interfaces.md` | API signatures, schemas | When writing any evaluation code | -| `patterns-evaluation.md` | Running evals, comparing | When executing evaluations | -| `patterns-scorers.md` | Custom scorer creation | When built-in scorers aren't enough | -| `patterns-datasets.md` | Dataset building | When preparing evaluation data | -| `patterns-trace-analysis.md` | Trace debugging | When analyzing agent behavior | -| `patterns-context-optimization.md` | Token/latency fixes | When agent is slow or expensive | -| `user-journeys.md` | High-level workflows | When starting a new evaluation project | - -## Critical API Facts - -- **Use:** `mlflow.genai.evaluate()` (NOT `mlflow.evaluate()`) -- **Data format:** `{"inputs": {"query": "..."}}` (nested structure required) -- **predict_fn:** Receives `**unpacked kwargs` (not a dict) - -See `GOTCHAS.md` for complete list. diff --git a/.claude/skills/mlflow-evaluation/references/user-journeys.md b/.claude/skills/mlflow-evaluation/references/user-journeys.md deleted file mode 100644 index 01cb4ccd..00000000 --- a/.claude/skills/mlflow-evaluation/references/user-journeys.md +++ /dev/null @@ -1,332 +0,0 @@ -# User Journey Guides - -Step-by-step workflows for common evaluation scenarios. - ---- - -## Journey 0: Strategy Alignment (ALWAYS START HERE) - -**Starting Point**: You need to evaluate an agent -**Goal**: Align on what to evaluate before writing any code - -**PRIORITY:** Before writing evaluation code, complete strategy alignment. This ensures evaluations measure what matters and provide actionable insights. - -### Step 1: Understand the Agent - -Before evaluating, gather context about what you're evaluating: - -**Questions to ask (or investigate in the codebase):** -1. **What does this agent do?** (data analysis, RAG, multi-turn chat, task automation) -2. **What tools does it use?** (UC functions, vector search, external APIs) -3. **What is the input/output format?** (messages format, structured output) -4. **What is the current state?** (prototype, production, needs improvement) - -**Actions to take:** -- Read the agent's main code file (e.g., `agent.py`) -- Review the config file for system prompts and tool definitions -- Check existing tests or evaluation scripts -- Look at CLAUDE.md or README for project context - -### Step 2: Align on What to Evaluate - -**Evaluation dimensions to consider:** - -| Dimension | When to Use | Example Scorer | -|-----------|-------------|----------------| -| **Safety** | Always (table stakes) | `Safety()` | -| **Correctness** | When ground truth exists | `Correctness()` | -| **Relevance** | When responses should address queries | `RelevanceToQuery()` | -| **Groundedness** | RAG systems with retrieved context | `RetrievalGroundedness()` | -| **Domain Guidelines** | Domain-specific requirements | `Guidelines(name="...", guidelines="...")` | -| **Format/Structure** | Structured output requirements | Custom scorer | -| **Tool Usage** | Agents with tool calls | Custom scorer checking tool selection | - -**Questions to ask the user:** -1. What are the **must-have** quality criteria? (safety, accuracy, relevance) -2. What are the **nice-to-have** criteria? (conciseness, tone, format) -3. Are there **specific failure modes** you've seen or worry about? -4. Do you have **ground truth** or expected answers for test cases? - -### Step 3: Define User Scenarios (Evaluation Dataset) - -**Types of test cases to include:** - -| Category | Purpose | Example | -|----------|---------|---------| -| **Happy Path** | Core functionality works | Typical user questions | -| **Edge Cases** | Boundary conditions | Empty inputs, very long queries | -| **Adversarial** | Robustness testing | Prompt injection, off-topic | -| **Multi-turn** | Conversation handling | Follow-up questions, context recall | -| **Domain-specific** | Business logic | Industry terminology, specific formats | - -**Questions to ask the user:** -1. What are the **most common** questions users ask? -2. What are **challenging** questions the agent should handle? -3. Are there questions it should **refuse** to answer? -4. Do you have **existing test cases** or production traces to start from? - -### Step 4: Establish Success Criteria - -**Define quality gates before running evaluation:** - -```python -QUALITY_GATES = { - "safety": 1.0, # 100% - non-negotiable - "correctness": 0.9, # 90% - high bar for accuracy - "relevance": 0.85, # 85% - good relevance - "concise": 0.8, # 80% - nice to have -} -``` - -**Questions to ask the user:** -1. What pass rates are **acceptable** for each dimension? -2. Which metrics are **blocking** vs **informational**? -3. How will evaluation results **inform decisions**? (ship/no-ship, iterate, investigate) - -### Strategy Alignment Checklist - -Before implementing evaluation, confirm: -- [ ] Agent purpose and architecture understood -- [ ] Evaluation dimensions agreed upon -- [ ] Test case categories identified -- [ ] Success criteria defined -- [ ] Data source identified (new, traces, existing dataset) - ---- - -## Journey 3: "Something Broke" - Regression Detection - -**Starting Point**: You made changes to your agent and suspect something regressed -**Goal**: Identify what broke and verify the fix - -### Steps - -1. **Establish baseline metrics** - ```bash - # Run evaluation on the previous version (or use saved baseline) - cd agents/tool_calling_dspy - python run_quick_eval.py - ``` - Record key metrics: `classifier_accuracy`, `tool_selection_accuracy`, `follows_instructions` - -2. **Run evaluation on current version** - ```bash - python run_quick_eval.py - ``` - -3. **Compare metrics** - ```python - from evaluation.optimization_history import OptimizationHistory - - history = OptimizationHistory() - print(history.compare_iterations(-2, -1)) # Compare last two - ``` - -4. **Identify regression source** - - If `classifier_accuracy` dropped → Check ClassifierSignature changes - - If `tool_selection_accuracy` dropped → Check tool descriptions, required_tools field - - If `follows_instructions` dropped → Check ExecutorSignature output format - -5. **Analyze failing traces** - ``` - /eval:analyze-traces [experiment-id] - ``` - Look for: - - Error patterns in specific test categories - - Tool call failures - - Unexpected outputs - -6. **Fix and re-evaluate** - - Revert problematic changes or apply targeted fix - - Re-run evaluation - - Verify metrics restored - -### Commands Used -- `python run_quick_eval.py` - Run evaluation -- `/eval:analyze-traces` - Deep trace analysis -- `OptimizationHistory.compare_iterations()` - Metric comparison - -### Success Indicators -- Metrics return to baseline or improve -- No new failing test cases -- Trace analysis shows expected behavior - ---- - -## Journey 7: "My Multi-Agent is Slow" - Performance Optimization - -**Starting Point**: Your agent responses are too slow -**Goal**: Identify bottlenecks and reduce latency - -### Steps - -1. **Run evaluation with latency scoring** - ```bash - cd agents/tool_calling_dspy - python run_quick_eval.py - ``` - Note the latency metrics: - - `classifier_latency_ms` - - `rewriter_latency_ms` - - `executor_latency_ms` - - `total_latency_ms` - -2. **Identify the bottleneck stage** - | Latency | Typical Range | If High, Check | - |---------|---------------|----------------| - | classifier_latency | <5s | ClassifierSignature verbosity | - | rewriter_latency | <10s | QueryRewriterSignature complexity | - | executor_latency | <30s | Tool call count, response generation | - -3. **Analyze traces for slow stages** - ``` - /eval:analyze-traces [experiment-id] - ``` - Focus on: - - Span durations by stage - - Number of LLM calls per stage - - Tool execution times - -4. **Run signature analysis** - ```bash - python -m evaluation.analyze_signatures - ``` - Look for: - - High total description chars (>2000) - - Verbose OutputField descriptions - - Missing examples (causes more retries) - -5. **Apply optimizations** - - **For high classifier latency:** - - Simplify ClassifierSignature docstring - - Add concrete examples to reduce ambiguity - - **For high executor latency:** - - Simplify ExecutorSignature.answer format - - Reduce output format requirements - - Consider caching repeated tool calls - - **For high total latency:** - - Review if all stages are necessary - - Consider parallel execution where possible - -6. **Re-evaluate and compare** - ```bash - python run_quick_eval.py - ``` - Use `OptimizationHistory.compare_iterations()` to verify improvement - -### Commands Used -- `python run_quick_eval.py` - Run evaluation with latency scoring -- `/eval:analyze-traces` - Trace analysis with timing breakdown -- `python -m evaluation.analyze_signatures` - Signature verbosity analysis - -### Success Indicators -- Target latencies: classifier <5s, executor <30s, total <60s -- No regression in accuracy metrics -- Consistent improvement across test categories - ---- - -## Journey 8: "Improve My Prompts" - Systematic Prompt Optimization - -**Starting Point**: Your agent works but could be more accurate -**Goal**: Systematically improve prompt quality through evaluation - -### Steps - -1. **Establish baseline** - ```bash - cd agents/tool_calling_dspy - python run_quick_eval.py - ``` - Record all metrics in `optimization_history.json` - -2. **Run signature analysis** - ```bash - python -m evaluation.analyze_signatures - ``` - Review the report for: - - Metric correlations (which signatures affect which metrics) - - Specific issues flagged per signature - -3. **Prioritize fixes by metric impact** - - | Metric | Primary Signature | Common Issues | - |--------|-------------------|---------------| - | follows_instructions | ExecutorSignature | Verbose answer format, unclear structure | - | tool_selection_accuracy | ClassifierSignature | No examples, ambiguous tool descriptions | - | classifier_accuracy | ClassifierSignature | Verbose docstring, unclear query_type mapping | - -4. **Apply ONE fix at a time** - - Make a single, targeted change - - Document the change in your commit message - - Track in optimization_history.json - -5. **Re-evaluate immediately** - ```bash - python run_quick_eval.py - ``` - - If improved → Keep change, move to next fix - - If regressed → Revert and try different approach - - If unchanged → Consider if fix was necessary - -6. **Iterate until targets met** - - | Metric | Target | - |--------|--------| - | classifier_accuracy | 95%+ | - | tool_selection_accuracy | 90%+ | - | follows_instructions | 80%+ | - -7. **Document successful optimizations** - ```python - from evaluation.optimization_history import OptimizationHistory - - history = OptimizationHistory() - print(history.summary()) - ``` - -### Commands Used -- `python run_quick_eval.py` - Run evaluation -- `python -m evaluation.analyze_signatures` - Identify prompt issues -- `/optimize:context --quick` - Full optimization loop (when endpoint available) - -### Success Indicators -- All target metrics met -- No regressions from baseline -- Clear documentation of what changed and why -- Optimization history shows positive trend - ---- - -## Quick Reference - -### Which Journey Am I On? - -| Symptom | Journey | -|---------|---------| -| "It was working before" | Journey 3 (Regression) | -| "It's too slow" | Journey 7 (Performance) | -| "It's not accurate enough" | Journey 8 (Prompt Optimization) | - -### Common Tools Across Journeys - -| Tool | Purpose | -|------|---------| -| `run_quick_eval.py` | Fast evaluation (8 test cases) | -| `run_full_eval.py` | Full evaluation (23 test cases) | -| `analyze_signatures.py` | Signature/prompt analysis | -| `OptimizationHistory` | Track iterations | -| `/eval:analyze-traces` | Deep trace analysis | -| `/optimize:context` | Full optimization loop | - -### Metric Targets - -| Metric | Target | Critical Threshold | -|--------|--------|-------------------| -| classifier_accuracy | 95%+ | <80% | -| tool_selection_accuracy | 90%+ | <70% | -| follows_instructions | 80%+ | <50% | -| executor_latency | <30s | >60s | diff --git a/.claude/skills/refresh-databricks-skills/SKILL.md b/.claude/skills/refresh-databricks-skills/SKILL.md new file mode 100644 index 00000000..47395dd4 --- /dev/null +++ b/.claude/skills/refresh-databricks-skills/SKILL.md @@ -0,0 +1,59 @@ +--- +name: refresh-databricks-skills +description: Use when Databricks skills need updating, user asks to refresh or sync skills from upstream, or skills seem outdated compared to the ai-dev-kit repo +--- + +# Refresh Databricks Skills + +## Overview + +Pulls the latest Databricks skills from the upstream source repo and replaces all existing Databricks skills in the project while preserving non-Databricks skills (e.g., superpowers workflow skills). + +**Source repo:** `https://github.com/databricks-solutions/ai-dev-kit` (path: `databricks-skills/`) + +## When to Use + +- User asks to update, refresh, or sync Databricks skills +- Skills seem outdated or missing newer Databricks features +- A new Databricks skill was added upstream that the project needs + +## Process + +1. **Clone the upstream repo** (shallow clone for speed): + ```bash + git clone --depth 1 https://github.com/databricks-solutions/ai-dev-kit.git $TMPDIR/ai-dev-kit + ``` + +2. **Identify non-Databricks skills to preserve.** These are the superpowers workflow skills that live alongside Databricks skills. List them by checking which directories in `.claude/skills/` do NOT have a matching folder in the upstream `databricks-skills/` directory. Common superpowers skills include: `brainstorming`, `dispatching-parallel-agents`, `executing-plans`, `finishing-a-development-branch`, `receiving-code-review`, `requesting-code-review`, `subagent-driven-development`, `systematic-debugging`, `test-driven-development`, `using-git-worktrees`, `using-superpowers`, `verification-before-completion`, `writing-plans`, `writing-skills`. Also preserve any other project-specific skills (like this one: `refresh-databricks-skills`). + +3. **Remove old Databricks skills** from `.claude/skills/`, keeping all non-Databricks skills identified above. + +4. **Copy new Databricks skills** from the cloned repo. Copy every directory under `databricks-skills/` except `TEMPLATE`: + ```bash + SKILLS_DIR=".claude/skills" + UPSTREAM="$TMPDIR/ai-dev-kit/databricks-skills" + for dir in "$UPSTREAM"/databricks-* "$UPSTREAM"/spark-*; do + [ -d "$dir" ] && cp -r "$dir" "$SKILLS_DIR/$(basename "$dir")" + done + ``` + +5. **Clean up** the cloned repo: + ```bash + rm -rf $TMPDIR/ai-dev-kit + ``` + +6. **Report** the count of skills added, removed, and updated. + +## After Refreshing + +If the project is deployed as a Databricks App, remind the user to sync the updated skills to the workspace and redeploy: +```bash +databricks workspace import-dir --overwrite --profile +databricks apps deploy --source-code-path --profile +``` + +## Common Mistakes + +- **Deleting non-Databricks skills:** Always identify and preserve superpowers and project-specific skills before removing anything. +- **Forgetting this skill itself:** `refresh-databricks-skills` must be preserved during the refresh. +- **Not using `--depth 1`:** Full clone is slow and unnecessary. Always shallow clone. diff --git a/.claude/skills/spark-declarative-pipelines/SKILL.md b/.claude/skills/spark-declarative-pipelines/SKILL.md deleted file mode 100644 index 6db988d3..00000000 --- a/.claude/skills/spark-declarative-pipelines/SKILL.md +++ /dev/null @@ -1,474 +0,0 @@ ---- -name: spark-declarative-pipelines -description: "Creates, configures, and updates Databricks Lakeflow Spark Declarative Pipelines (SDP/LDP) using serverless compute. Handles streaming tables, materialized views, CDC, SCD Type 2, and Auto Loader ingestion patterns. Use when building data pipelines, working with Delta Live Tables, ingesting streaming data, implementing change data capture, or when the user mentions SDP, LDP, DLT, Lakeflow pipelines, streaming tables, or bronze/silver/gold medallion architectures." ---- - -# Lakeflow Spark Declarative Pipelines (SDP) - -## Quick Reference - -| Concept | Details | -|---------|---------| -| **Names** | SDP = Spark Declarative Pipelines = LDP = Lakeflow Declarative Pipelines = Lakeflow Pipelines (all interchangeable) | -| **Python Import** | `from pyspark import pipelines as dp` | -| **Primary Decorators** | `@dp.table()`, `@dp.materialized_view()` | -| **Replaces** | Delta Live Tables (DLT) with `import dlt` | -| **Based On** | Apache Spark 4.1+ (Databricks' modern data pipeline framework) | -| **Docs** | https://docs.databricks.com/aws/en/ldp/developer/python-dev | - ---- - -## Official Documentation - -- **[Lakeflow Spark Declarative Pipelines Overview](https://docs.databricks.com/aws/en/ldp/)** - Main documentation hub -- **[SQL Language Reference](https://docs.databricks.com/aws/en/ldp/developer/sql-dev)** - SQL syntax for streaming tables and materialized views -- **[Python Language Reference](https://docs.databricks.com/aws/en/ldp/developer/python-ref)** - `pyspark.pipelines` API -- **[Loading Data](https://docs.databricks.com/aws/en/ldp/load)** - Auto Loader, Kafka, Kinesis ingestion -- **[Change Data Capture (CDC)](https://docs.databricks.com/aws/en/ldp/cdc)** - AUTO CDC, SCD Type 1/2 -- **[Developing Pipelines](https://docs.databricks.com/aws/en/ldp/develop)** - File structure, testing, validation -- **[Liquid Clustering](https://docs.databricks.com/aws/en/delta/clustering)** - Modern data layout optimization - ---- - -## Quick Start: Initialize New Pipeline Project - -**RECOMMENDED**: Use `databricks pipelines init` to create production-ready Asset Bundle projects with multi-environment support. - -### When to Use Bundle Initialization - -Use bundle initialization for **New pipeline projects** for a professional structure from the start - -Use manual workflow for: -- Quick prototyping without multi-environment needs -- Existing manual projects you want to continue -- Learning/experimentation - -### Step 1: Initialize Project - -I will automatically run this command when you request a new pipeline: - -```bash -databricks pipelines init -``` - -**Interactive Prompts:** -- **Project name**: e.g., `customer_orders_pipeline` -- **Initial catalog**: Unity Catalog name (e.g., `main`, `prod_catalog`) -- **Personal schema per user?**: `yes` for dev (each user gets their own schema), `no` for prod -- **Language**: SQL or Python (auto-detected from your request - see language detection below) - -**Generated Structure:** -``` -my_pipeline/ -├── databricks.yml # Multi-environment config (dev/prod) -├── resources/ -│ └── *_etl.pipeline.yml # Pipeline resource definition -└── src/ - └── *_etl/ - ├── explorations/ # Exploratory code in .ipynb - └── transformations/ # Your .sql or .py files here -``` - -### Step 2: Customize Transformations - -Replace the example code created by the init process with custom transformation files in `src/transformations/` based on provided requirements, using best practice guidance from this skill. - - -### Step 3: Deploy and Run - -```bash -# Deploy to workspace (dev by default) -databricks bundle deploy - -# Run pipeline -databricks bundle run my_pipeline_etl - -# Deploy to production -databricks bundle deploy --target prod -``` - -I can run these commands for you using the Bash tool. - -**For medallion architecture** (bronze/silver/gold), two approaches work: -- **Flat with naming** (template default): `bronze_*.sql`, `silver_*.sql`, `gold_*.sql` -- **Subdirectories**: `bronze/orders.sql`, `silver/cleaned.sql`, `gold/summary.sql` - -Both work with the `transformations/**` glob pattern. Choose based on preference. - -See **[8-project-initialization.md](8-project-initialization.md)** for complete details on bundle initialization, migration, and troubleshooting. - ---- - -## Alternative: Manual Workflow (Advanced) - -For rapid prototyping, experimentation, or when you prefer direct control without Asset Bundles, use the manual workflow with MCP tools. - -Use MCP tools to create, run, and iterate on **serverless SDP pipelines**. The **primary tool is `create_or_update_pipeline`** which handles the entire lifecycle. - -**IMPORTANT: Always create serverless pipelines (default).** Only use classic clusters if user explicitly requires R language, Spark RDD APIs, or JAR libraries. - -### Step 1: Write Pipeline Files Locally - -Create `.sql` or `.py` files in a local folder: - -``` -my_pipeline/ -├── bronze/ -│ ├── ingest_orders.sql # SQL (default for most cases) -│ └── ingest_events.py # Python (for complex logic) -├── silver/ -│ └── clean_orders.sql -└── gold/ - └── daily_summary.sql -``` - -**SQL Example** (`bronze/ingest_orders.sql`): -```sql -CREATE OR REFRESH STREAMING TABLE bronze_orders -CLUSTER BY (order_date) -AS -SELECT - *, - current_timestamp() AS _ingested_at, - _metadata.file_path AS _source_file -FROM read_files( - '/Volumes/catalog/schema/raw/orders/', - format => 'json', - schemaHints => 'order_id STRING, customer_id STRING, amount DECIMAL(10,2), order_date DATE' -); -``` - -**Python Example** (`bronze/ingest_events.py`): -```python -from pyspark import pipelines as dp -from pyspark.sql.functions import col, current_timestamp - -@dp.table(name="bronze_events", cluster_by=["event_date"]) -def bronze_events(): - return ( - spark.readStream.format("cloudFiles") - .option("cloudFiles.format", "json") - .load("/Volumes/catalog/schema/raw/events/") - .withColumn("_ingested_at", current_timestamp()) - .withColumn("_source_file", col("_metadata.file_path")) - ) -``` - -**Language Selection:** -- **Auto-detection**: I analyze your request for keywords: - - **SQL indicators**: "SQL", "sql files", "simple transformations", "aggregations", "materialized view", "CREATE OR REFRESH" - - **Python indicators**: "Python", ".py files", "UDF", "complex logic", "ML inference", "external API", "@dp.table", "pandas" -- **Prompt for clarification** when language intent is unclear or mixed -- **Use SQL** for: Transformations, aggregations, filtering, joins (most cases) -- **Generate ONE language** per request unless you explicitly ask for mixed pipeline - -See **[8-project-initialization.md](8-project-initialization.md)** for detailed language detection logic. - -### Step 2: Upload to Databricks Workspace - -```python -# MCP Tool: upload_folder -upload_folder( - local_folder="/path/to/my_pipeline", - workspace_folder="/Workspace/Users/user@example.com/my_pipeline" -) -``` - -### Step 3: Create/Update and Run Pipeline - -Use **`create_or_update_pipeline`** - the main entry point. It: -1. Searches for an existing pipeline with the same name (or uses `id` from `extra_settings`) -2. Creates a new pipeline or updates the existing one -3. Optionally starts a pipeline run -4. Optionally waits for completion and returns detailed results - -```python -# MCP Tool: create_or_update_pipeline -result = create_or_update_pipeline( - name="my_orders_pipeline", - root_path="/Workspace/Users/user@example.com/my_pipeline", - catalog="my_catalog", - schema="my_schema", - workspace_file_paths=[ - "/Workspace/Users/user@example.com/my_pipeline/bronze/ingest_orders.sql", - "/Workspace/Users/user@example.com/my_pipeline/silver/clean_orders.sql", - "/Workspace/Users/user@example.com/my_pipeline/gold/daily_summary.sql" - ], - start_run=True, # Start immediately - wait_for_completion=True, # Wait and return final status - full_refresh=True, # Full refresh all tables - timeout=1800 # 30 minute timeout -) -``` - -**Result contains actionable information:** -```python -{ - "success": True, # Did the operation succeed? - "pipeline_id": "abc-123", # Pipeline ID for follow-up operations - "pipeline_name": "my_orders_pipeline", - "created": True, # True if new, False if updated - "state": "COMPLETED", # COMPLETED, FAILED, TIMEOUT, etc. - "catalog": "my_catalog", # Target catalog - "schema": "my_schema", # Target schema - "duration_seconds": 45.2, # Time taken - "message": "Pipeline created and completed successfully in 45.2s. Tables written to my_catalog.my_schema", - "error_message": None, # Error summary if failed - "errors": [] # Detailed error list if failed -} -``` - -### Step 4: Handle Results - -**On Success:** -```python -if result["success"]: - # Verify output tables - stats = get_table_details( - catalog="my_catalog", - schema="my_schema", - table_names=["bronze_orders", "silver_orders", "gold_daily_summary"] - ) -``` - -**On Failure:** -```python -if not result["success"]: - # Message includes suggested next steps - print(result["message"]) - # "Pipeline created but run failed. State: FAILED. Error: Column 'amount' not found. - # Use get_pipeline_events(pipeline_id='abc-123') for full details." - - # Get detailed errors - events = get_pipeline_events(pipeline_id=result["pipeline_id"], max_results=50) -``` - -### Step 5: Iterate Until Working - -1. Review errors from result or `get_pipeline_events` -2. Fix issues in local files -3. Re-upload with `upload_folder` -4. Run `create_or_update_pipeline` again (it will update, not recreate) -5. Repeat until `result["success"] == True` - ---- - -## Quick Reference: MCP Tools - -### Primary Tool - -| Tool | Description | -|------|-------------| -| **`create_or_update_pipeline`** | **Main entry point.** Creates or updates pipeline, optionally runs and waits. Returns detailed status with `success`, `state`, `errors`, and actionable `message`. | - -### Pipeline Management - -| Tool | Description | -|------|-------------| -| `find_pipeline_by_name` | Find existing pipeline by name, returns pipeline_id | -| `get_pipeline` | Get pipeline configuration and current state | -| `start_update` | Start pipeline run (`validate_only=True` for dry run) | -| `get_update` | Poll update status (QUEUED, RUNNING, COMPLETED, FAILED) | -| `stop_pipeline` | Stop a running pipeline | -| `get_pipeline_events` | Get error messages for debugging failed runs | -| `delete_pipeline` | Delete a pipeline | - -### Supporting Tools - -| Tool | Description | -|------|-------------| -| `upload_folder` | Upload local folder to workspace (parallel) | -| `get_table_details` | Verify output tables have expected schema and row counts | -| `execute_sql` | Run ad-hoc SQL to inspect data | - ---- - -## Reference Documentation (Local) - -Load these for detailed patterns: - -- **[1-ingestion-patterns.md](1-ingestion-patterns.md)** - Auto Loader, Kafka, Event Hub, Kinesis, file formats -- **[2-streaming-patterns.md](2-streaming-patterns.md)** - Deduplication, windowing, stateful operations, joins -- **[3-scd-patterns.md](3-scd-patterns.md)** - Querying SCD Type 2 history tables, temporal joins -- **[4-performance-tuning.md](4-performance-tuning.md)** - Liquid Clustering, optimization, state management -- **[5-python-api.md](5-python-api.md)** - Modern `dp` API vs legacy `dlt` API comparison -- **[6-dlt-migration.md](6-dlt-migration.md)** - Migrating existing DLT pipelines to SDP -- **[7-advanced-configuration.md](7-advanced-configuration.md)** - `extra_settings` parameter reference and examples -- **[8-project-initialization.md](8-project-initialization.md)** - Using `databricks pipelines init`, Asset Bundles, language detection, and migration guides - ---- - -## Best Practices (2025) - -### Project Structure -- **Default to `databricks pipelines init`** for new projects (creates Asset Bundle) -- **Use Asset Bundles** for multi-environment deployments (dev/staging/prod) -- **Manual structure only** for quick prototypes or legacy migration -- **Medallion architecture**: Two approaches work with Asset Bundles: - - **Flat structure** (template default): `bronze_*.sql`, `silver_*.sql`, `gold_*.sql` in `transformations/` - - **Subdirectories**: `transformations/bronze/`, `transformations/silver/`, `transformations/gold/` - - Both work with the `transformations/**` glob pattern - choose based on team preference -- See **[8-project-initialization.md](8-project-initialization.md)** for project setup details - - -### Modern Defaults -- **CLUSTER BY** (Liquid Clustering), not PARTITION BY - see [4-performance-tuning.md](4-performance-tuning.md) -- **Raw `.sql`/`.py` files**, not notebooks -- **Serverless compute ONLY** - Do not use classic clusters unless explicitly required -- **Unity Catalog** (required for serverless) -- **read_files()** for cloud storage ingestion - see [1-ingestion-patterns.md](1-ingestion-patterns.md) - -### Reading Tables in Python - -**Modern SDP Best Practice:** -- Use `spark.read.table()` for batch reads -- Use `spark.readStream.table()` for streaming reads -- Don't use `dp.read()` or `dp.read_stream()` (old syntax, no longer documented) -- Don't use `dlt.read()` or `dlt.read_stream()` (legacy DLT API) - -**Key Point:** SDP automatically tracks table dependencies from standard Spark DataFrame operations. No special read APIs are needed. - -#### Three-Tier Identifier Resolution - -SDP supports three levels of table name qualification: - -| Level | Syntax | When to Use | -|-------|--------|-------------| -| **Unqualified** | `spark.read.table("my_table")` | Reading tables within the same pipeline's target catalog/schema (recommended) | -| **Partially-qualified** | `spark.read.table("other_schema.my_table")` | Reading from different schema in same catalog | -| **Fully-qualified** | `spark.read.table("other_catalog.other_schema.my_table")` | Reading from external catalogs/schemas | - -#### Option 1: Unqualified Names (Recommended for Pipeline Tables) - -**Best practice for tables within the same pipeline.** SDP resolves unqualified names to the pipeline's configured target catalog and schema. This makes code portable across environments (dev/prod). - -```python -@dp.table(name="silver_clean") -def silver_clean(): - # Reads from pipeline's target catalog/schema (e.g., dev_catalog.dev_schema.bronze_raw) - return ( - spark.read.table("bronze_raw") - .filter(F.col("valid") == True) - ) - -@dp.table(name="silver_events") -def silver_events(): - # Streaming read from same pipeline's bronze_events table - return ( - spark.readStream.table("bronze_events") - .withColumn("processed_at", F.current_timestamp()) - ) -``` - -#### Option 2: Pipeline Parameters (For External Sources) - -**Use `spark.conf.get()` to parameterize external catalog/schema references.** Define parameters in pipeline configuration, then reference them at the module level. - -```python -from pyspark import pipelines as dp -from pyspark.sql import functions as F - -# Get parameterized values at module level (evaluated once at pipeline start) -source_catalog = spark.conf.get("source_catalog") -source_schema = spark.conf.get("source_schema", "sales") # with default - -@dp.table(name="transaction_summary") -def transaction_summary(): - return ( - spark.read.table(f"{source_catalog}.{source_schema}.transactions") - .groupBy("account_id") - .agg( - F.count("txn_id").alias("txn_count"), - F.sum("txn_amount").alias("account_revenue") - ) - ) -``` - -**Configure parameters in pipeline settings:** -- **Asset Bundles**: Add to `pipeline.yml` under `configuration:` -- **Manual/MCP**: Pass via `extra_settings.configuration` dict - -```yaml -# In resources/my_pipeline.pipeline.yml -configuration: - source_catalog: "shared_catalog" - source_schema: "sales" -``` - -#### Option 3: Fully-Qualified Names (For Fixed External References) - -Use when referencing specific external tables that don't change across environments: - -```python -@dp.table(name="enriched_orders") -def enriched_orders(): - # Pipeline-internal table (unqualified) - orders = spark.read.table("bronze_orders") - - # External reference table (fully-qualified) - products = spark.read.table("shared_catalog.reference.products") - - return orders.join(products, "product_id") -``` - -#### Choosing the Right Approach - -| Scenario | Recommended Approach | -|----------|---------------------| -| Reading tables created in same pipeline | **Unqualified names** - portable, uses target catalog/schema | -| Reading from external source that varies by environment | **Pipeline parameters** - configurable per deployment | -| Reading from shared/reference tables with fixed location | **Fully-qualified names** - explicit and clear | -| Mixed pipeline (some internal, some external) | **Combine approaches** - unqualified for internal, parameters for external | - ---- - -## Common Issues - -| Issue | Solution | -|-------|----------| -| **Empty output tables** | Use `get_table_details` to verify, check upstream sources | -| **Pipeline stuck INITIALIZING** | Normal for serverless, wait a few minutes | -| **"Column not found"** | Check `schemaHints` match actual data | -| **Streaming reads fail** | Use `FROM STREAM(table)` for streaming sources | -| **Timeout during run** | Increase `timeout`, or use `wait_for_completion=False` and poll with `get_update` | -| **MV doesn't refresh** | Enable row tracking on source tables | -| **SCD2 schema errors** | Let SDP infer START_AT/END_AT columns | - -**For detailed errors**, the `result["message"]` from `create_or_update_pipeline` includes suggested next steps. Use `get_pipeline_events(pipeline_id=...)` for full stack traces. - ---- - -## Advanced Pipeline Configuration - -For advanced configuration options (development mode, continuous pipelines, custom clusters, notifications, Python dependencies, etc.), see **[7-advanced-configuration.md](7-advanced-configuration.md)**. - ---- - -## Platform Constraints - -### Serverless Pipeline Requirements (Default) -| Requirement | Details | -|-------------|---------| -| **Unity Catalog** | Required - serverless pipelines always use UC | -| **Workspace Region** | Must be in serverless-enabled region | -| **Serverless Terms** | Must accept serverless terms of use | -| **CDC Features** | Requires serverless (or Pro/Advanced with classic clusters) | - -### Serverless Limitations (When Classic Clusters Required) -| Limitation | Workaround | -|------------|-----------| -| **R language** | Not supported - use classic clusters if required | -| **Spark RDD APIs** | Not supported - use classic clusters if required | -| **JAR libraries** | Not supported - use classic clusters if required | -| **Maven coordinates** | Not supported - use classic clusters if required | -| **DBFS root access** | Limited - must use Unity Catalog external locations | -| **Global temp views** | Not supported | - -### General Constraints -| Constraint | Details | -|------------|---------| -| **Schema Evolution** | Streaming tables require full refresh for incompatible changes | -| **SQL Limitations** | PIVOT clause unsupported | -| **Sinks** | Python only, streaming only, append flows only | - -**Default to serverless** unless user explicitly requires R, RDD APIs, or JAR libraries. \ No newline at end of file diff --git a/.claude/skills/spark-python-data-source/SKILL.md b/.claude/skills/spark-python-data-source/SKILL.md new file mode 100644 index 00000000..898b9d2b --- /dev/null +++ b/.claude/skills/spark-python-data-source/SKILL.md @@ -0,0 +1,311 @@ +--- +name: spark-python-data-source +description: Use when building custom Spark data source connectors for external systems (databases, APIs, message queues), implementing batch/streaming readers/writers, or creating data source plugins for systems without native Spark support. Triggers - "build Spark data source", "create Spark connector", "implement Spark reader/writer", "connect Spark to [system]", "streaming data source" +--- + +# spark-python-data-source + +Build custom Python data sources for Apache Spark 4.0+ to read from and write to external systems in batch and streaming modes. + +## When to use + +Use when building Spark connectors for external systems that lack native support: +- External databases, APIs, message queues +- Custom file formats or protocols +- Real-time streaming data sources +- Systems requiring specialized authentication or protocols + +Triggers: "build Spark data source", "create Spark connector", "implement Spark reader/writer", "connect Spark to [system]", "streaming data source" + +## Instructions + +You are an experienced Spark developer building custom Python data sources following the PySpark DataSource API. Follow these principles and patterns: + +### Core Architecture + +Each data source follows a flat, single-level inheritance structure: + +1. **DataSource class** - Entry point returning readers/writers +2. **Base Reader/Writer classes** - Shared logic for options and data processing +3. **Batch classes** - Inherit from base + `DataSourceReader`/`DataSourceWriter` +4. **Stream classes** - Inherit from base + `DataSourceStreamReader`/`DataSourceStreamWriter` + +### Critical Design Principles + +**SIMPLE over CLEVER** - These are non-negotiable: + +✅ REQUIRED: +- Flat single-level inheritance only +- Direct implementations, no abstractions +- Explicit imports, explicit control flow +- Standard library first, minimal dependencies +- Simple classes with single responsibilities + +❌ FORBIDDEN: +- Abstract base classes or complex inheritance +- Factory patterns or dependency injection +- Decorators for cross-cutting concerns +- Complex configuration classes +- Async/await (unless absolutely necessary) +- Connection pooling or caching (unless critical) +- Generic "framework" code +- Premature optimization + +### Implementation Pattern + +```python +from pyspark.sql.datasource import ( + DataSource, DataSourceReader, DataSourceWriter, + DataSourceStreamReader, DataSourceStreamWriter +) + +# 1. DataSource class +class YourDataSource(DataSource): + @classmethod + def name(cls): + return "your-format" + + def __init__(self, options): + self.options = options + + def schema(self): + return self._infer_or_return_schema() + + def reader(self, schema): + return YourBatchReader(self.options, schema) + + def streamReader(self, schema): + return YourStreamReader(self.options, schema) + + def writer(self, schema, overwrite): + return YourBatchWriter(self.options, schema) + + def streamWriter(self, schema, overwrite): + return YourStreamWriter(self.options, schema) + +# 2. Base Writer with shared logic +class YourWriter: + def __init__(self, options, schema=None): + # Validate required options + self.url = options.get("url") + assert self.url, "url is required" + self.batch_size = int(options.get("batch_size", "50")) + self.schema = schema + + def write(self, iterator): + # Import libraries here for partition execution + import requests + from pyspark import TaskContext + + context = TaskContext.get() + partition_id = context.partitionId() + + msgs = [] + cnt = 0 + + for row in iterator: + cnt += 1 + msgs.append(row.asDict()) + + if len(msgs) >= self.batch_size: + self._send_batch(msgs) + msgs = [] + + if msgs: + self._send_batch(msgs) + + return SimpleCommitMessage(partition_id=partition_id, count=cnt) + + def _send_batch(self, msgs): + # Implement send logic + pass + +# 3. Batch Writer +class YourBatchWriter(YourWriter, DataSourceWriter): + pass + +# 4. Stream Writer +class YourStreamWriter(YourWriter, DataSourceStreamWriter): + def commit(self, messages, batchId): + pass + + def abort(self, messages, batchId): + pass + +# 5. Base Reader with partitioning +class YourReader: + def __init__(self, options, schema): + self.url = options.get("url") + assert self.url, "url is required" + self.schema = schema + + def partitions(self): + # Return list of partitions for parallel reading + return [YourPartition(0, start, end)] + + def read(self, partition): + # Import here for executor execution + import requests + + response = requests.get(f"{self.url}?start={partition.start}") + for item in response.json(): + yield tuple(item.values()) + +# 6. Batch Reader +class YourBatchReader(YourReader, DataSourceReader): + pass + +# 7. Stream Reader +class YourStreamReader(YourReader, DataSourceStreamReader): + def initialOffset(self): + return {"offset": "0"} + + def latestOffset(self): + return {"offset": str(self._get_latest())} + + def partitions(self, start, end): + return [YourPartition(0, start["offset"], end["offset"])] + + def commit(self, end): + pass +``` + +### Project Setup + +```bash +# Create project +poetry new your-datasource +cd your-datasource +poetry add pyspark pytest pytest-spark + +# Development commands - CRITICAL: Always use 'poetry run' +poetry run pytest # Run tests +poetry run ruff check src/ # Lint +poetry run ruff format src/ # Format +poetry build # Build wheel +``` + +### Registration and Usage + +```python +# Register +from your_package import YourDataSource +spark.dataSource.register(YourDataSource) + +# Batch read +df = spark.read.format("your-format").option("url", "...").load() + +# Batch write +df.write.format("your-format").option("url", "...").save() + +# Streaming read +df = spark.readStream.format("your-format").option("url", "...").load() + +# Streaming write +df.writeStream.format("your-format").option("url", "...").start() +``` + +### Key Implementation Decisions + +**Partitioning Strategy**: Choose based on data source characteristics +- Time-based: For APIs with temporal data (see [partitioning-patterns.md](references/partitioning-patterns.md)) +- Token-range: For distributed databases (see [partitioning-patterns.md](references/partitioning-patterns.md)) +- ID-range: For paginated APIs + +**Authentication**: Support multiple methods in priority order +- Databricks Unity Catalog credentials +- Cloud default credentials (managed identity) +- Explicit credentials (service principal, API key, username/password) +- See [authentication-patterns.md](references/authentication-patterns.md) + +**Type Conversion**: Map between Spark and external types +- Handle nulls, timestamps, UUIDs, collections +- See [type-conversion.md](references/type-conversion.md) + +**Streaming Offsets**: Design for exactly-once semantics +- JSON-serializable offset class +- Non-overlapping partition boundaries +- See [streaming-patterns.md](references/streaming-patterns.md) + +**Error Handling**: Implement retries and resilience +- Exponential backoff for retryable errors +- Circuit breakers for cascading failures +- See [error-handling.md](references/error-handling.md) + +### Testing Approach + +```python +import pytest +from unittest.mock import patch, Mock + +@pytest.fixture +def spark(): + from pyspark.sql import SparkSession + return SparkSession.builder.master("local[2]").getOrCreate() + +def test_data_source_name(): + assert YourDataSource.name() == "your-format" + +def test_writer_sends_data(spark): + with patch('requests.post') as mock_post: + mock_post.return_value = Mock(status_code=200) + + df = spark.createDataFrame([(1, "test")], ["id", "value"]) + df.write.format("your-format").option("url", "http://api").save() + + assert mock_post.called +``` + +### Code Review Checklist + +Before implementing, ask: +1. Is this the simplest way to solve this problem? +2. Would a new developer understand this immediately? +3. Am I adding abstraction for real needs vs hypothetical flexibility? +4. Can I solve this with standard library? +5. Does this follow the established flat pattern? + +### Common Mistakes to Avoid + +- Creating abstract base classes for "reusability" +- Adding configuration frameworks or dependency injection +- Premature optimization before measuring performance +- Complex error handling hierarchies +- Importing heavy libraries at module level (import in methods) +- Using `python` command directly (always use `poetry run`) + +### Reference Implementations + +Study these for real-world patterns: +- [cyber-spark-data-connectors](https://github.com/alexott/cyber-spark-data-connectors) - Sentinel, Splunk, REST +- [spark-cassandra-data-source](https://github.com/alexott/spark-cassandra-data-source) - Token-range partitioning +- [pyspark-hubspot](https://github.com/dgomez04/pyspark-hubspot) - REST API pagination +- [pyspark-mqtt](https://github.com/databricks-industry-solutions/python-data-sources/tree/main/mqtt) - Streaming with TLS + +## Usage + +``` +Create a Spark data source for reading from MongoDB with sharding support +Build a streaming connector for RabbitMQ with at-least-once delivery +Implement a batch writer for Snowflake with staged uploads +Write a data source for REST API with OAuth2 authentication and pagination +``` + +## Related + +- databricks-testing: Test data sources on Databricks clusters +- databricks-spark-declarative-pipelines: Use custom sources in DLT pipelines +- python-dev: Python development best practices + +## References + +- [partitioning-patterns.md](references/partitioning-patterns.md) - Parallel reading strategies +- [authentication-patterns.md](references/authentication-patterns.md) - Multi-method auth implementations +- [type-conversion.md](references/type-conversion.md) - Bidirectional type mapping +- [streaming-patterns.md](references/streaming-patterns.md) - Offset management and watermarking +- [error-handling.md](references/error-handling.md) - Retries, circuit breakers, resilience +- [testing-patterns.md](references/testing-patterns.md) - Unit and integration testing +- [production-patterns.md](references/production-patterns.md) - Observability, security, validation +- [Official Databricks Documentation](https://docs.databricks.com/aws/en/pyspark/datasources) +- [Apache Spark Python DataSource Tutorial](https://spark.apache.org/docs/latest/api/python/tutorial/sql/python_data_source.html) +- [awesome-python-datasources](https://github.com/allisonwang-db/awesome-python-datasources) - directory of available implementations. diff --git a/.claude/skills/spark-python-data-source/references/authentication-patterns.md b/.claude/skills/spark-python-data-source/references/authentication-patterns.md new file mode 100644 index 00000000..700f516e --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/authentication-patterns.md @@ -0,0 +1,361 @@ +# Authentication Patterns + +Multi-method authentication strategies with clear priority ordering. + +## Priority-Based Authentication + +Support multiple authentication methods with fallback: + +```python +class AuthenticatedDataSource(DataSource): + def __init__(self, options): + # Priority 1: Databricks Unity Catalog credential + self.databricks_credential = options.get("databricks_credential") + + # Priority 2: Cloud default credential (managed identity) + self.default_credential = options.get("default_credential", "false").lower() == "true" + + # Priority 3: Service principal + self.tenant_id = options.get("tenant_id") + self.client_id = options.get("client_id") + self.client_secret = options.get("client_secret") + + # Priority 4: API key + self.api_key = options.get("api_key") + + # Priority 5: Username/password + self.username = options.get("username") + self.password = options.get("password") + + # Validate at least one method is configured + self._validate_auth() + + def _validate_auth(self): + """Validate at least one auth method is configured.""" + has_databricks_cred = bool(self.databricks_credential) + has_default_cred = self.default_credential + has_service_principal = all([self.tenant_id, self.client_id, self.client_secret]) + has_api_key = bool(self.api_key) + has_basic_auth = bool(self.username and self.password) + + if not any([has_databricks_cred, has_default_cred, has_service_principal, + has_api_key, has_basic_auth]): + raise AssertionError( + "Authentication required. Provide one of: " + "'databricks_credential', 'default_credential=true', " + "'tenant_id/client_id/client_secret', 'api_key', or 'username/password'" + ) +``` + +## Azure Authentication + +### Unity Catalog Service Credential + +```python +def _get_azure_credential_uc(credential_name): + """Get credential from Unity Catalog.""" + import databricks.service_credentials + + return databricks.service_credentials.getServiceCredentialsProvider(credential_name) +``` + +### Default Credential (Managed Identity) + +```python +def _get_azure_credential_default(authority=None): + """Get DefaultAzureCredential for managed identity.""" + from azure.identity import DefaultAzureCredential + + if authority: + return DefaultAzureCredential(authority=authority) + return DefaultAzureCredential() +``` + +### Service Principal + +```python +def _get_azure_credential_sp(tenant_id, client_id, client_secret, authority=None): + """Get service principal credential.""" + from azure.identity import ClientSecretCredential + + if authority: + return ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret, + authority=authority + ) + return ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret + ) +``` + +### Multi-Cloud Support + +```python +def _get_azure_cloud_config(cloud_name): + """Get cloud-specific endpoints and authorities.""" + from azure.identity import AzureAuthorityHosts + + cloud_configs = { + "public": (None, None), + "government": ( + AzureAuthorityHosts.AZURE_GOVERNMENT, + "https://api.loganalytics.us" + ), + "china": ( + AzureAuthorityHosts.AZURE_CHINA, + "https://api.loganalytics.azure.cn" + ), + } + + cloud = (cloud_name or "public").lower().strip() + + if cloud not in cloud_configs: + valid = ", ".join(cloud_configs.keys()) + raise ValueError(f"Invalid cloud '{cloud_name}'. Valid: {valid}") + + return cloud_configs[cloud] + +def _create_azure_client_with_cloud(options): + """Create Azure client with cloud-specific configuration.""" + cloud_name = options.get("azure_cloud", "public") + authority, endpoint = _get_azure_cloud_config(cloud_name) + + # Get credential based on priority + credential = _get_credential(options, authority) + + # Create client with cloud-specific endpoint + from azure.monitor.query import LogsQueryClient + + if endpoint: + return LogsQueryClient(credential, endpoint=endpoint) + return LogsQueryClient(credential) +``` + +## API Key Authentication + +### Header-Based + +```python +def _get_api_key_auth(api_key): + """Get API key authentication headers.""" + return {"Authorization": f"Bearer {api_key}"} + +def _create_session_with_api_key(api_key): + """Create requests session with API key.""" + import requests + + session = requests.Session() + session.headers.update({"Authorization": f"Bearer {api_key}"}) + return session +``` + +### Query Parameter-Based + +```python +def _build_url_with_api_key(base_url, api_key): + """Add API key as query parameter.""" + from urllib.parse import urlencode + + params = {"api_key": api_key} + return f"{base_url}?{urlencode(params)}" +``` + +## Basic Authentication + +```python +def _get_basic_auth(username, password): + """Get HTTP Basic Auth.""" + from requests.auth import HTTPBasicAuth + return HTTPBasicAuth(username, password) + +def _create_session_with_basic_auth(username, password): + """Create session with basic auth.""" + import requests + + session = requests.Session() + session.auth = (username, password) + return session +``` + +## OAuth2 Authentication + +### Client Credentials Flow + +```python +def _get_oauth2_token(token_url, client_id, client_secret, scope): + """Get OAuth2 token using client credentials.""" + import requests + + response = requests.post( + token_url, + data={ + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": client_secret, + "scope": scope + } + ) + response.raise_for_status() + + return response.json()["access_token"] + +class OAuth2Writer: + def __init__(self, options): + self.token_url = options["token_url"] + self.client_id = options["client_id"] + self.client_secret = options["client_secret"] + self.scope = options.get("scope", "") + self._token = None + self._token_expiry = None + + def _get_valid_token(self): + """Get valid token, refresh if expired.""" + from datetime import datetime, timedelta + + if not self._token or datetime.now() >= self._token_expiry: + self._token = _get_oauth2_token( + self.token_url, + self.client_id, + self.client_secret, + self.scope + ) + # Assume 1 hour expiry if not provided + self._token_expiry = datetime.now() + timedelta(hours=1) + + return self._token + + def write(self, iterator): + """Write with OAuth2 authentication.""" + import requests + + token = self._get_valid_token() + headers = {"Authorization": f"Bearer {token}"} + + for row in iterator: + requests.post(self.url, json=row.asDict(), headers=headers) +``` + +## Complete Authentication Factory + +```python +def get_credential(options): + """ + Get credential based on configuration priority. + + Priority: + 1. databricks_credential + 2. default_credential + 3. Service principal (tenant_id/client_id/client_secret) + 4. API key + 5. Username/password + """ + + # Priority 1: Databricks credential + if options.get("databricks_credential"): + import databricks.service_credentials + return databricks.service_credentials.getServiceCredentialsProvider( + options["databricks_credential"] + ) + + # Priority 2: Cloud default credential + if options.get("default_credential", "false").lower() == "true": + authority = options.get("authority") + if authority: + from azure.identity import DefaultAzureCredential + return DefaultAzureCredential(authority=authority) + from azure.identity import DefaultAzureCredential + return DefaultAzureCredential() + + # Priority 3: Service principal + if all(k in options for k in ["tenant_id", "client_id", "client_secret"]): + from azure.identity import ClientSecretCredential + authority = options.get("authority") + if authority: + return ClientSecretCredential( + tenant_id=options["tenant_id"], + client_id=options["client_id"], + client_secret=options["client_secret"], + authority=authority + ) + return ClientSecretCredential( + tenant_id=options["tenant_id"], + client_id=options["client_id"], + client_secret=options["client_secret"] + ) + + # Priority 4: API key + if "api_key" in options: + return {"Authorization": f"Bearer {options['api_key']}"} + + # Priority 5: Basic auth + if "username" in options and "password" in options: + from requests.auth import HTTPBasicAuth + return HTTPBasicAuth(options["username"], options["password"]) + + raise ValueError("No valid authentication method configured") +``` + +## Security Best Practices + +### Never Log Sensitive Values + +```python +class SecureDataSource(DataSource): + def __init__(self, options): + self._sensitive_keys = { + "password", "api_key", "client_secret", "token", "access_token" + } + + # Store actual values + self.options = options + + # Create sanitized version for logging + self._safe_options = self._sanitize_options(options) + + def _sanitize_options(self, options): + """Mask sensitive values for logging.""" + safe = {} + for key, value in options.items(): + if key.lower() in self._sensitive_keys: + safe[key] = "***REDACTED***" + else: + safe[key] = value + return safe + + def __repr__(self): + return f"SecureDataSource({self._safe_options})" +``` + +### Use Secrets Management + +```python +def _load_secrets_from_dbutils(scope, keys): + """Load secrets from Databricks secrets.""" + try: + from pyspark.dbutils import DBUtils + from pyspark.sql import SparkSession + + spark = SparkSession.getActiveSession() + dbutils = DBUtils(spark) + + secrets = {} + for key in keys: + secrets[key] = dbutils.secrets.get(scope=scope, key=key) + + return secrets + + except Exception as e: + raise ValueError(f"Failed to load secrets from scope '{scope}': {e}") + +# Usage +if "secret_scope" in options: + secrets = _load_secrets_from_dbutils( + options["secret_scope"], + ["password", "api_key"] + ) + options.update(secrets) +``` diff --git a/.claude/skills/spark-python-data-source/references/error-handling.md b/.claude/skills/spark-python-data-source/references/error-handling.md new file mode 100644 index 00000000..01bbf2f9 --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/error-handling.md @@ -0,0 +1,432 @@ +# Error Handling and Resilience + +Patterns for retries, circuit breakers, and graceful degradation. + +## Exponential Backoff + +Retry with exponential backoff for transient failures: + +```python +def write_with_retry(self, iterator): + """Write with exponential backoff.""" + import time + + max_retries = int(self.options.get("max_retries", "5")) + initial_backoff = float(self.options.get("initial_backoff", "1.0")) + max_backoff = float(self.options.get("max_backoff", "60.0")) + + for row in iterator: + retry_count = 0 + + while retry_count <= max_retries: + try: + self._send_data(row) + break # Success + + except Exception as e: + if not self._is_retryable_error(e): + # Non-retryable error - fail immediately + raise + + if retry_count >= max_retries: + # Max retries exceeded + raise Exception(f"Max retries ({max_retries}) exceeded: {e}") + + # Calculate backoff with exponential growth + backoff = min(initial_backoff * (2 ** retry_count), max_backoff) + time.sleep(backoff) + retry_count += 1 + +def _is_retryable_error(self, error): + """Determine if error is retryable.""" + from requests.exceptions import RequestException, Timeout, ConnectionError + + # Network errors are retryable + if isinstance(error, (Timeout, ConnectionError)): + return True + + # HTTP errors + if hasattr(error, 'response') and error.response: + status_code = error.response.status_code + # Retry on 429 (throttling) and 5xx (server errors) + if status_code == 429 or 500 <= status_code < 600: + return True + + return False +``` + +## Retry with Throttling Respect + +Handle API rate limiting with Retry-After header: + +```python +def write_with_throttling(self, iterator): + """Write with respect for rate limits.""" + import time + from requests.exceptions import HTTPError + + for row in iterator: + max_attempts = 5 + attempt = 0 + + while attempt < max_attempts: + try: + self._send_data(row) + break + + except HTTPError as e: + if e.response.status_code == 429: + # Throttled - respect Retry-After header + retry_after = self._get_retry_after(e.response) + time.sleep(retry_after) + attempt += 1 + else: + raise + + if attempt >= max_attempts: + raise Exception("Max retry attempts for throttling exceeded") + +def _get_retry_after(self, response): + """Extract retry delay from Retry-After header.""" + retry_after = response.headers.get("Retry-After") + + if retry_after: + try: + # Try as seconds (int) + return int(retry_after) + except ValueError: + # Try as HTTP date + from datetime import datetime + try: + retry_date = datetime.strptime(retry_after, "%a, %d %b %Y %H:%M:%S GMT") + delay = (retry_date - datetime.utcnow()).total_seconds() + return max(0, delay) + except ValueError: + pass + + # Default fallback + return 1.0 +``` + +## Circuit Breaker + +Prevent cascading failures with circuit breaker pattern: + +```python +class CircuitBreaker: + """Circuit breaker to prevent cascading failures.""" + + def __init__(self, threshold=10, timeout=300): + self.threshold = threshold # failures before opening + self.timeout = timeout # seconds before trying again + self.consecutive_failures = 0 + self.circuit_open = False + self.circuit_open_until = None + + def record_success(self): + """Record successful operation.""" + self.consecutive_failures = 0 + + def record_failure(self): + """Record failed operation.""" + from datetime import datetime, timedelta + + self.consecutive_failures += 1 + + if self.consecutive_failures >= self.threshold: + self.circuit_open = True + self.circuit_open_until = datetime.now() + timedelta(seconds=self.timeout) + + def is_open(self): + """Check if circuit is open.""" + from datetime import datetime + + if self.circuit_open: + if datetime.now() >= self.circuit_open_until: + # Timeout expired - try again + self.circuit_open = False + self.consecutive_failures = 0 + return False + return True + + return False + +class ResilientWriter: + def __init__(self, options): + self.circuit_breaker = CircuitBreaker( + threshold=int(options.get("circuit_breaker_threshold", "10")), + timeout=int(options.get("circuit_breaker_timeout", "300")) + ) + + def write(self, iterator): + """Write with circuit breaker protection.""" + for row in iterator: + if self.circuit_breaker.is_open(): + raise Exception("Circuit breaker open - too many failures") + + try: + self._send_data(row) + self.circuit_breaker.record_success() + + except Exception as e: + self.circuit_breaker.record_failure() + raise +``` + +## Graceful Degradation + +Handle partial failures and fallback strategies: + +```python +def read_with_fallback(self, partition): + """Read with fallback to secondary sources.""" + try: + # Try primary source + yield from self._read_primary(partition) + + except ConnectionError as e: + # Primary failed - try secondary + if self.secondary_endpoint: + print(f"Primary failed, using secondary: {e}") + yield from self._read_secondary(partition) + else: + raise + + except TimeoutError as e: + # Timeout - try with smaller partitions + if partition.can_subdivide(): + print(f"Timeout, subdividing: {e}") + for sub_partition in partition.subdivide(): + yield from self.read(sub_partition) + else: + raise + + except PartialResultError as e: + # Partial results - log warning and continue + print(f"Warning: Partial results for partition {partition.id}: {e}") + yield from e.partial_results +``` + +## Bulk Operation Error Handling + +Handle errors in bulk operations: + +```python +def write_batch_with_error_handling(self, iterator): + """Write in batches with individual error tracking.""" + from cassandra.concurrent import execute_concurrent_with_args + + batch_size = int(self.options.get("batch_size", "1000")) + fail_on_first_error = self.options.get("fail_on_first_error", "true").lower() == "true" + + batch_params = [] + failed_rows = [] + + for row in iterator: + batch_params.append(self._row_to_params(row)) + + if len(batch_params) >= batch_size: + # Execute batch + results = execute_concurrent_with_args( + self.session, + self.prepared_statement, + batch_params, + concurrency=100, + raise_on_first_error=fail_on_first_error + ) + + # Check for failures + for success, result_or_error in results: + if not success: + failed_rows.append((batch_params[i], result_or_error)) + + batch_params = [] + + # Final batch + if batch_params: + results = execute_concurrent_with_args( + self.session, + self.prepared_statement, + batch_params, + concurrency=100, + raise_on_first_error=fail_on_first_error + ) + + for i, (success, result_or_error) in enumerate(results): + if not success: + failed_rows.append((batch_params[i], result_or_error)) + + # Handle failed rows + if failed_rows: + if fail_on_first_error: + raise Exception(f"{len(failed_rows)} rows failed to write") + else: + # Log failures but continue + print(f"Warning: {len(failed_rows)} rows failed to write") +``` + +## Dead Letter Queue + +Store failed records for later processing: + +```python +class DeadLetterQueueWriter: + """Writer with dead letter queue for failed records.""" + + def __init__(self, options): + self.dlq_path = options.get("dlq_path") + self.dlq_enabled = bool(self.dlq_path) + + def write(self, iterator): + """Write with DLQ support.""" + from datetime import datetime + import json + + successful = 0 + failed = 0 + + for row in iterator: + try: + self._send_data(row) + successful += 1 + + except Exception as e: + failed += 1 + + if self.dlq_enabled: + self._write_to_dlq(row, e) + else: + raise + + return { + "successful": successful, + "failed": failed + } + + def _write_to_dlq(self, row, error): + """Write failed record to dead letter queue.""" + from datetime import datetime + import json + import os + + dlq_record = { + "timestamp": datetime.now().isoformat(), + "error": str(error), + "error_type": type(error).__name__, + "row": row.asDict() + } + + # Append to DLQ file + os.makedirs(os.path.dirname(self.dlq_path), exist_ok=True) + + with open(self.dlq_path, 'a') as f: + f.write(json.dumps(dlq_record) + '\n') +``` + +## Timeout Handling + +Enforce operation timeouts: + +```python +import signal +from contextlib import contextmanager + +class TimeoutError(Exception): + pass + +def timeout_handler(signum, frame): + raise TimeoutError("Operation timed out") + +@contextmanager +def timeout(seconds): + """Context manager for operation timeout.""" + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + +class TimeoutWriter: + def write(self, iterator): + """Write with per-row timeout.""" + timeout_seconds = int(self.options.get("write_timeout", "30")) + + for row in iterator: + try: + with timeout(timeout_seconds): + self._send_data(row) + + except TimeoutError: + print(f"Write timeout after {timeout_seconds}s") + raise +``` + +## Error Aggregation + +Collect and report errors systematically: + +```python +class ErrorAggregator: + """Aggregate errors for batch reporting.""" + + def __init__(self): + self.errors = [] + self.error_counts = {} + + def record_error(self, error, context=None): + """Record an error with context.""" + error_type = type(error).__name__ + error_msg = str(error) + + self.errors.append({ + "type": error_type, + "message": error_msg, + "context": context + }) + + # Count by type + self.error_counts[error_type] = self.error_counts.get(error_type, 0) + 1 + + def get_summary(self): + """Get error summary.""" + return { + "total_errors": len(self.errors), + "by_type": self.error_counts, + "sample_errors": self.errors[:10] # First 10 + } + +class ErrorAwareWriter: + def write(self, iterator): + """Write with error aggregation.""" + aggregator = ErrorAggregator() + successful = 0 + + for i, row in enumerate(iterator): + try: + self._send_data(row) + successful += 1 + + except Exception as e: + aggregator.record_error(e, context={"row_index": i}) + + # Report summary + if aggregator.errors: + summary = aggregator.get_summary() + print(f"Completed with {successful} success, {summary['total_errors']} errors") + print(f"Error breakdown: {summary['by_type']}") + + if summary['total_errors'] > successful: + raise Exception(f"Too many errors: {summary}") +``` + +## Best Practices + +1. **Retry Only Transient Errors**: Don't retry client errors (4xx) +2. **Respect Rate Limits**: Use Retry-After headers and backoff +3. **Circuit Breakers**: Prevent cascading failures in distributed systems +4. **Timeout Operations**: Set reasonable timeouts to prevent hangs +5. **Log Errors**: Capture error context for debugging +6. **Dead Letter Queues**: Store failed records for later analysis +7. **Monitor Failure Rates**: Alert on anomalous error rates +8. **Graceful Degradation**: Continue with partial results when appropriate diff --git a/.claude/skills/spark-python-data-source/references/partitioning-patterns.md b/.claude/skills/spark-python-data-source/references/partitioning-patterns.md new file mode 100644 index 00000000..699e75a5 --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/partitioning-patterns.md @@ -0,0 +1,319 @@ +# Partitioning Patterns + +Strategies for distributing reads across Spark executors for parallel processing. + +## Time-Based Partitioning + +For APIs with temporal data or streaming sources. + +### Fixed Duration Partitions + +```python +from pyspark.sql.datasource import InputPartition +from datetime import datetime, timedelta + +class TimeRangePartition(InputPartition): + def __init__(self, start_time, end_time): + self.start_time = start_time + self.end_time = end_time + +class TimeBasedReader: + def __init__(self, options, schema): + self.partition_duration = int(options.get("partition_duration", "3600")) # seconds + # Parse start/end time from options + + def partitions(self): + """Split time range into fixed-duration partitions.""" + partitions = [] + current = self.start_time + delta = timedelta(seconds=self.partition_duration) + + while current < self.end_time: + next_time = min(current + delta, self.end_time) + partitions.append(TimeRangePartition(current, next_time)) + current = next_time + + return partitions + + def read(self, partition): + """Query data for specific time range.""" + response = self._query_api( + start=partition.start_time, + end=partition.end_time + ) + for item in response: + yield self._convert_to_row(item) +``` + +### Auto-Subdividing for Large Results + +Handle APIs with result size limits by automatically subdividing large partitions: + +```python +class AutoSubdivideReader: + def __init__(self, options, schema): + self.min_partition_seconds = int(options.get("min_partition_seconds", "60")) + self.max_retries = int(options.get("max_retries", "5")) + + def read(self, partition): + """Read with automatic subdivision on size limit errors.""" + try: + response = self._execute_query(partition.start_time, partition.end_time) + + # Check if response is partial due to size limits + if self._is_size_limit_error(response): + yield from self._read_with_subdivision(partition) + return + + yield from self._process_response(response) + + except Exception as e: + raise + + def _read_with_subdivision(self, partition): + """Recursively subdivide large partitions.""" + duration = (partition.end_time - partition.start_time).total_seconds() + + if duration <= self.min_partition_seconds: + raise Exception( + f"Cannot subdivide further. Duration {duration}s at minimum. " + f"Consider more selective query or increase min_partition_seconds." + ) + + # Split in half + midpoint = partition.start_time + timedelta(seconds=duration / 2) + + first_half = TimeRangePartition(partition.start_time, midpoint) + second_half = TimeRangePartition(midpoint, partition.end_time) + + yield from self.read(first_half) + yield from self.read(second_half) + + def _is_size_limit_error(self, response): + """Detect result size limit errors.""" + size_limit_codes = [ + "QueryExecutionResultSizeLimitExceeded", + "ResponsePayloadTooLarge", + "E_QUERY_RESULT_SET_TOO_LARGE", + ] + + if hasattr(response, "error") and response.error: + if response.error.code in size_limit_codes: + return True + + error_str = str(response.error).lower() + return any(p in error_str for p in ["size limit", "too large", "exceed"]) + + return False +``` + +## Token-Range Partitioning + +For distributed databases using consistent hashing (Cassandra, ScyllaDB). + +### Cassandra Token-Range Pattern + +```python +from collections import namedtuple + +class TokenRangePartition(InputPartition): + def __init__(self, partition_id, start_token, end_token, pk_columns, + is_wrap_around=False, min_token=None): + self.partition_id = partition_id + self.start_token = start_token # None = unbounded + self.end_token = end_token # None = unbounded + self.pk_columns = pk_columns + self.is_wrap_around = is_wrap_around + self.min_token = min_token + +class TokenRangeReader: + def _get_token_ranges(self, token_map): + """Compute token ranges from cluster token ring.""" + if not token_map or not token_map.ring: + return [] + + TokenRange = namedtuple('TokenRange', ['start', 'end']) + ranges = [] + ring = sorted(token_map.ring) + + for i in range(len(ring)): + start = ring[i] + end = ring[(i + 1) % len(ring)] # Wrap around + ranges.append(TokenRange(start=start, end=end)) + + return ranges + + def partitions(self): + """Create partitions following TokenRangesScan.java logic.""" + if not self.token_ranges: + return [] + + partitions = [] + sorted_ranges = sorted(self.token_ranges) + partition_id = 0 + + min_token_obj = sorted_ranges[0].start + min_token = min_token_obj.value if hasattr(min_token_obj, 'value') else str(min_token_obj) + + for i, token_range in enumerate(sorted_ranges): + start_value = token_range.start.value if hasattr(token_range.start, 'value') else str(token_range.start) + end_value = token_range.end.value if hasattr(token_range.end, 'value') else str(token_range.end) + + if start_value == end_value: + # Case 1: Single-node cluster (entire ring) + partition = TokenRangePartition( + partition_id=partition_id, + start_token=min_token, + end_token=None, # Unbounded + pk_columns=self.pk_columns, + is_wrap_around=True, + min_token=min_token + ) + partitions.append(partition) + partition_id += 1 + + elif i == 0: + # Case 2: First range - split into TWO partitions + # Partition 1: token <= minToken (wrap-around) + partition1 = TokenRangePartition( + partition_id=partition_id, + start_token=None, + end_token=min_token, + pk_columns=self.pk_columns, + is_wrap_around=True, + min_token=min_token + ) + partitions.append(partition1) + partition_id += 1 + + # Partition 2: token > start AND token <= end + partition2 = TokenRangePartition( + partition_id=partition_id, + start_token=start_value, + end_token=end_value, + pk_columns=self.pk_columns, + is_wrap_around=False, + min_token=min_token + ) + partitions.append(partition2) + partition_id += 1 + + elif end_value == min_token: + # Case 3: Range ending at minToken - no upper bound + partition = TokenRangePartition( + partition_id=partition_id, + start_token=start_value, + end_token=None, + pk_columns=self.pk_columns, + is_wrap_around=False, + min_token=min_token + ) + partitions.append(partition) + partition_id += 1 + + else: + # Case 4: Normal range - both bounds + partition = TokenRangePartition( + partition_id=partition_id, + start_token=start_value, + end_token=end_value, + pk_columns=self.pk_columns, + is_wrap_around=False, + min_token=min_token + ) + partitions.append(partition) + partition_id += 1 + + return partitions + + def read(self, partition): + """Build query with token range predicates.""" + pk_cols_str = ", ".join(partition.pk_columns) + + # Build WHERE clause based on bounds + if partition.start_token is None: + where_clause = f"token({pk_cols_str}) <= {partition.end_token}" + elif partition.end_token is None: + where_clause = f"token({pk_cols_str}) > {partition.start_token}" + else: + where_clause = ( + f"token({pk_cols_str}) > {partition.start_token} AND " + f"token({pk_cols_str}) <= {partition.end_token}" + ) + + query = f"SELECT {columns} FROM {table} WHERE {where_clause}" + + # Execute and yield results + for row in self._execute_query(query): + yield row +``` + +## ID-Range Partitioning + +For APIs with pagination or sequential IDs. + +```python +class IdRangePartition(InputPartition): + def __init__(self, partition_id, start_id, end_id): + self.partition_id = partition_id + self.start_id = start_id + self.end_id = end_id + +class IdRangeReader: + def __init__(self, options, schema): + self.num_partitions = int(options.get("num_partitions", "4")) + self.page_size = int(options.get("page_size", "1000")) + + def partitions(self): + """Split by ID ranges.""" + # Get total count from API + total = self._get_total_count() + partition_size = total // self.num_partitions + + partitions = [] + for i in range(self.num_partitions): + start_id = i * partition_size + end_id = (i + 1) * partition_size if i < self.num_partitions - 1 else total + partitions.append(IdRangePartition(i, start_id, end_id)) + + return partitions + + def read(self, partition): + """Paginate through ID range.""" + current_id = partition.start_id + + while current_id < partition.end_id: + response = self._query_api( + start_id=current_id, + limit=self.page_size + ) + + for item in response.items: + yield self._convert_to_row(item) + + current_id += self.page_size +``` + +## Partition Count Guidelines + +**For Batch Reads:** +- Start with 2-4x number of executor cores +- Adjust based on data volume and partition size +- Consider external system load limits + +**For Streaming Reads:** +- Use fixed-duration partitions (e.g., 1 hour) +- Let Spark handle parallelism across micro-batches +- Balance latency vs throughput + +**For Token-Range:** +- One partition per token range (determined by cluster) +- Naturally distributes based on data distribution +- May split first range into two partitions + +## Performance Considerations + +1. **Partition Size**: Aim for 128MB - 1GB per partition +2. **API Rate Limits**: Respect rate limits with concurrency controls +3. **Network Overhead**: Larger partitions reduce round-trips +4. **Skew Handling**: Monitor for data skew, repartition if needed diff --git a/.claude/skills/spark-python-data-source/references/production-patterns.md b/.claude/skills/spark-python-data-source/references/production-patterns.md new file mode 100644 index 00000000..71928ca7 --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/production-patterns.md @@ -0,0 +1,475 @@ +# Production Patterns + +Observability, security, validation, and operational best practices. + +## Observability and Metrics + +Track operation metrics for monitoring: + +```python +class ObservableWriter: + """Writer with comprehensive metrics tracking.""" + + def write(self, iterator): + """Write with metrics collection.""" + from pyspark import TaskContext + from datetime import datetime + import time + + context = TaskContext.get() + partition_id = context.partitionId() + + metrics = { + "partition_id": partition_id, + "rows_processed": 0, + "rows_failed": 0, + "bytes_sent": 0, + "batches_sent": 0, + "retry_count": 0, + "start_time": time.time(), + "errors": [] + } + + try: + for row in iterator: + try: + size = self._send_row(row) + metrics["rows_processed"] += 1 + metrics["bytes_sent"] += size + + except Exception as e: + metrics["rows_failed"] += 1 + metrics["errors"].append({ + "type": type(e).__name__, + "message": str(e) + }) + + if not self.continue_on_error: + raise + + metrics["duration_seconds"] = time.time() - metrics["start_time"] + self._report_metrics(metrics) + + return SimpleCommitMessage( + partition_id=partition_id, + count=metrics["rows_processed"] + ) + + except Exception as e: + metrics["fatal_error"] = str(e) + self._report_failure(partition_id, metrics) + raise + + def _report_metrics(self, metrics): + """Report metrics to monitoring system.""" + # Example: CloudWatch, Prometheus, Databricks metrics + print(f"METRICS: {json.dumps(metrics)}") + + # Calculate derived metrics + if metrics["duration_seconds"] > 0: + throughput = metrics["rows_processed"] / metrics["duration_seconds"] + print(f"Throughput: {throughput:.2f} rows/second") +``` + +## Logging Best Practices + +Structured logging for production debugging: + +```python +import logging +import json + +# Configure structured logging +logging.basicConfig( + format='%(asctime)s %(levelname)s [%(name)s] %(message)s', + level=logging.INFO +) +logger = logging.getLogger(__name__) + +class StructuredLogger: + """Logger with structured output.""" + + @staticmethod + def log_operation(operation, context, **kwargs): + """Log operation with structured context.""" + log_entry = { + "operation": operation, + "context": context, + **kwargs + } + logger.info(json.dumps(log_entry)) + + @staticmethod + def log_error(operation, error, context): + """Log error with context.""" + log_entry = { + "operation": operation, + "error_type": type(error).__name__, + "error_message": str(error), + "context": context + } + logger.error(json.dumps(log_entry)) + +class LoggingWriter: + def write(self, iterator): + """Write with structured logging.""" + from pyspark import TaskContext + + context = TaskContext.get() + partition_id = context.partitionId() + + StructuredLogger.log_operation( + "write_start", + {"partition_id": partition_id} + ) + + try: + count = 0 + for row in iterator: + self._send_data(row) + count += 1 + + StructuredLogger.log_operation( + "write_complete", + {"partition_id": partition_id}, + rows_written=count + ) + + except Exception as e: + StructuredLogger.log_error( + "write_failed", + e, + {"partition_id": partition_id} + ) + raise +``` + +## Security Validation + +Input validation and sanitization: + +```python +import re +import ipaddress + +class SecureDataSource: + """Data source with security validation.""" + + # Sensitive keys that should never be logged + SENSITIVE_KEYS = { + "password", "api_key", "client_secret", "token", + "access_token", "refresh_token", "bearer_token" + } + + def __init__(self, options): + # Validate and sanitize options + self._validate_options(options) + self.options = options + + # Create sanitized version for logging + self._safe_options = self._sanitize_for_logging(options) + + def _validate_options(self, options): + """Comprehensive option validation.""" + # Validate required options + required = ["host", "database", "table"] + missing = [opt for opt in required if opt not in options] + if missing: + raise ValueError(f"Missing required options: {', '.join(missing)}") + + # Validate host (IP or hostname) + self._validate_host(options["host"]) + + # Validate port range + if "port" in options: + port = int(options["port"]) + if port < 1 or port > 65535: + raise ValueError(f"Port must be 1-65535, got {port}") + + # Validate table name (prevent SQL injection) + self._validate_identifier(options["table"], "table") + + # Validate numeric options + if "batch_size" in options: + batch_size = int(options["batch_size"]) + if batch_size < 1 or batch_size > 10000: + raise ValueError(f"batch_size must be 1-10000, got {batch_size}") + + def _validate_host(self, host): + """Validate host is valid IP or hostname.""" + try: + # Try as IP address + ipaddress.ip_address(host) + return + except ValueError: + pass + + # Validate as hostname + if not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9-\.]*[a-zA-Z0-9]$', host): + raise ValueError(f"Invalid host format: {host}") + + def _validate_identifier(self, identifier, name): + """Validate SQL identifier (table, column name).""" + # Prevent SQL injection + if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', identifier): + raise ValueError( + f"Invalid {name} identifier: {identifier}. " + f"Must contain only letters, numbers, and underscores, " + f"and start with a letter or underscore." + ) + + def _sanitize_for_logging(self, options): + """Mask sensitive values for logging.""" + safe = {} + for key, value in options.items(): + if key.lower() in self.SENSITIVE_KEYS: + safe[key] = "***REDACTED***" + else: + safe[key] = value + return safe + + def __repr__(self): + return f"SecureDataSource({self._safe_options})" +``` + +## Secrets Management + +Load credentials from secure storage: + +```python +def load_secrets_from_databricks(scope, keys): + """Load secrets from Databricks secrets.""" + try: + from pyspark.dbutils import DBUtils + from pyspark.sql import SparkSession + + spark = SparkSession.getActiveSession() + if not spark: + raise ValueError("No active Spark session") + + dbutils = DBUtils(spark) + secrets = {} + + for key in keys: + try: + secrets[key] = dbutils.secrets.get(scope=scope, key=key) + except Exception as e: + raise ValueError(f"Failed to load secret '{key}' from scope '{scope}': {e}") + + return secrets + + except Exception as e: + raise ValueError(f"Failed to access Databricks secrets: {e}") + +class SecureCredentialLoader: + """Load credentials securely.""" + + @staticmethod + def load_credentials(options): + """Load credentials from secure storage.""" + # Priority 1: Databricks secrets + if "secret_scope" in options: + secret_keys = [ + "username", "password", "api_key", "client_secret" + ] + secrets = load_secrets_from_databricks( + options["secret_scope"], + secret_keys + ) + options.update(secrets) + + # Priority 2: Environment variables + elif options.get("use_env_vars", "false").lower() == "true": + import os + options["username"] = os.environ.get("DB_USERNAME") + options["password"] = os.environ.get("DB_PASSWORD") + + return options +``` + +## Configuration Validation + +Validate configuration before execution: + +```python +class ConfigValidator: + """Validate data source configuration.""" + + VALID_CONSISTENCY_LEVELS = { + "ONE", "TWO", "THREE", "QUORUM", "ALL", + "LOCAL_QUORUM", "EACH_QUORUM", "LOCAL_ONE" + } + + VALID_COMPRESSION = { + "none", "gzip", "snappy", "lz4", "zstd" + } + + @classmethod + def validate(cls, options): + """Validate all configuration options.""" + errors = [] + + # Validate consistency level + if "consistency" in options: + consistency = options["consistency"].upper() + if consistency not in cls.VALID_CONSISTENCY_LEVELS: + errors.append( + f"Invalid consistency level '{consistency}'. " + f"Valid: {', '.join(cls.VALID_CONSISTENCY_LEVELS)}" + ) + + # Validate compression + if "compression" in options: + compression = options["compression"].lower() + if compression not in cls.VALID_COMPRESSION: + errors.append( + f"Invalid compression '{compression}'. " + f"Valid: {', '.join(cls.VALID_COMPRESSION)}" + ) + + # Validate numeric ranges + if "timeout" in options: + timeout = int(options["timeout"]) + if timeout < 0 or timeout > 300: + errors.append(f"timeout must be 0-300 seconds, got {timeout}") + + if "batch_size" in options: + batch_size = int(options["batch_size"]) + if batch_size < 1 or batch_size > 10000: + errors.append(f"batch_size must be 1-10000, got {batch_size}") + + # Validate dependent options + if options.get("ssl_enabled", "false").lower() == "true": + if "ssl_ca_cert" not in options: + errors.append("ssl_ca_cert required when ssl_enabled=true") + + if errors: + raise ValueError("Configuration errors:\n" + "\n".join(f"- {e}" for e in errors)) +``` + +## Resource Cleanup + +Ensure proper resource cleanup: + +```python +class ManagedResourceWriter: + """Writer with guaranteed resource cleanup.""" + + def __init__(self, options): + self.options = options + self._connection = None + self._session = None + + def _get_connection(self): + """Lazy connection initialization.""" + if self._connection is None: + self._connection = self._create_connection() + return self._connection + + def write(self, iterator): + """Write with guaranteed cleanup.""" + try: + connection = self._get_connection() + + for row in iterator: + self._send_data(connection, row) + + finally: + # Always cleanup resources + self._cleanup() + + def _cleanup(self): + """Clean up resources.""" + if self._session: + try: + self._session.close() + except Exception as e: + logger.warning(f"Error closing session: {e}") + finally: + self._session = None + + if self._connection: + try: + self._connection.close() + except Exception as e: + logger.warning(f"Error closing connection: {e}") + finally: + self._connection = None + + def __del__(self): + """Cleanup on garbage collection.""" + self._cleanup() +``` + +## Health Checks + +Monitor system health: + +```python +class HealthCheckMixin: + """Mixin for health check functionality.""" + + def check_health(self): + """Perform health check before operations.""" + checks = { + "connection": self._check_connection(), + "authentication": self._check_authentication(), + "rate_limit": self._check_rate_limit(), + "disk_space": self._check_disk_space() + } + + failed = [name for name, passed in checks.items() if not passed] + + if failed: + raise Exception(f"Health check failed: {', '.join(failed)}") + + return checks + + def _check_connection(self): + """Check connection to external system.""" + try: + self._test_connection() + return True + except Exception as e: + logger.error(f"Connection check failed: {e}") + return False + + def _check_authentication(self): + """Check authentication is valid.""" + try: + self._verify_credentials() + return True + except Exception as e: + logger.error(f"Authentication check failed: {e}") + return False + + def _check_rate_limit(self): + """Check if under rate limits.""" + # Check current rate usage + current_rate = self._get_current_rate() + limit = self._get_rate_limit() + + return current_rate < limit * 0.8 # 80% threshold + + def _check_disk_space(self): + """Check available disk space.""" + import shutil + + usage = shutil.disk_usage("/") + free_percent = (usage.free / usage.total) * 100 + + return free_percent > 10 # 10% minimum +``` + +## Operational Best Practices + +1. **Monitoring**: Track throughput, latency, error rates +2. **Logging**: Use structured logging with correlation IDs +3. **Secrets**: Never log sensitive values, use secrets management +4. **Validation**: Validate all inputs to prevent injection attacks +5. **Resource Cleanup**: Always close connections and clean up resources +6. **Health Checks**: Verify system health before operations +7. **Rate Limiting**: Respect API rate limits with backoff +8. **Alerting**: Set up alerts for error rates and latency +9. **Documentation**: Document all configuration options +10. **Version Control**: Tag releases and maintain changelog diff --git a/.claude/skills/spark-python-data-source/references/streaming-patterns.md b/.claude/skills/spark-python-data-source/references/streaming-patterns.md new file mode 100644 index 00000000..6f00ddd6 --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/streaming-patterns.md @@ -0,0 +1,400 @@ +# Streaming Patterns + +Offset management and streaming implementation patterns for exactly-once semantics. + +## Basic Offset Implementation + +Simple JSON-serializable offset: + +```python +class SimpleOffset: + """Basic offset with single timestamp field.""" + + def __init__(self, timestamp): + self.timestamp = timestamp + + def json(self): + """Serialize to JSON string.""" + import json + return json.dumps({"timestamp": self.timestamp}) + + @staticmethod + def from_json(json_str): + """Deserialize from JSON string.""" + import json + data = json.loads(json_str) + return SimpleOffset(data["timestamp"]) +``` + +## Multi-Field Offset + +Complex offset with multiple fields: + +```python +class MultiFieldOffset: + """Offset with timestamp, sequence ID, and partition.""" + + def __init__(self, timestamp, sequence_id, partition_id): + self.timestamp = timestamp + self.sequence_id = sequence_id + self.partition_id = partition_id + + def json(self): + import json + return json.dumps({ + "timestamp": self.timestamp, + "sequence_id": self.sequence_id, + "partition_id": self.partition_id + }) + + @staticmethod + def from_json(json_str): + import json + data = json.loads(json_str) + return MultiFieldOffset( + timestamp=data["timestamp"], + sequence_id=data["sequence_id"], + partition_id=data["partition_id"] + ) + + def __lt__(self, other): + """Enable offset comparison for ordering.""" + if self.timestamp != other.timestamp: + return self.timestamp < other.timestamp + if self.sequence_id != other.sequence_id: + return self.sequence_id < other.sequence_id + return self.partition_id < other.partition_id +``` + +## Stream Reader Implementation + +Complete streaming reader with offset management: + +```python +from pyspark.sql.datasource import DataSourceStreamReader + +class YourStreamReader(DataSourceStreamReader): + def __init__(self, options, schema): + super().__init__(options, schema) + + # Parse start time option + start_time = options.get("start_time", "latest") + + if start_time == "latest": + from datetime import datetime, timezone + self.start_time = datetime.now(timezone.utc).isoformat() + + elif start_time == "earliest": + # Query for earliest timestamp (one-time cost) + self.start_time = self._get_earliest_timestamp() + + else: + # Validate ISO 8601 format + from datetime import datetime + datetime.fromisoformat(start_time.replace("Z", "+00:00")) + self.start_time = start_time + + # Partition duration (e.g., 1 hour) + self.partition_duration = int(options.get("partition_duration", "3600")) + + def _get_earliest_timestamp(self): + """Find earliest data timestamp for 'earliest' option.""" + from datetime import datetime, timezone + + timestamp_column = self.options.get("timestamp_column", "timestamp") + query = f"{self.query} | summarize earliest=min({timestamp_column})" + + response = self._execute_query(query, timespan=None) + + if response.tables and response.tables[0].rows: + earliest_value = response.tables[0].rows[0][0] + if earliest_value: + if isinstance(earliest_value, datetime): + return earliest_value.isoformat() + return str(earliest_value) + + # Fallback to current time + return datetime.now(timezone.utc).isoformat() + + def initialOffset(self): + """ + Return initial offset (start time minus 1 microsecond). + + Subtract 1µs to compensate for +1µs in partitions() method, + preventing overlap between batches. + """ + from datetime import datetime, timedelta + + start_dt = datetime.fromisoformat(self.start_time.replace("Z", "+00:00")) + adjusted = start_dt - timedelta(microseconds=1) + return SimpleOffset(adjusted.isoformat()).json() + + def latestOffset(self): + """Return latest offset (current time).""" + from datetime import datetime, timezone + + current_time = datetime.now(timezone.utc).isoformat() + return SimpleOffset(current_time).json() + + def partitions(self, start, end): + """ + Create non-overlapping partitions for offset range. + + Adds 1µs to start to prevent overlap with previous batch. + """ + from datetime import datetime, timedelta + + start_offset = SimpleOffset.from_json(start) + end_offset = SimpleOffset.from_json(end) + + start_time = datetime.fromisoformat(start_offset.timestamp.replace("Z", "+00:00")) + end_time = datetime.fromisoformat(end_offset.timestamp.replace("Z", "+00:00")) + + # Add 1µs to prevent overlap with previous batch + # This works with -1µs in initialOffset() to ensure: + # - Initial batch: (start - 1µs) + 1µs = start (correct) + # - Subsequent batches: previous_end + 1µs (no overlap) + start_time = start_time + timedelta(microseconds=1) + + # Create fixed-duration partitions + partitions = [] + current = start_time + delta = timedelta(seconds=self.partition_duration) + + while current < end_time: + next_time = min(current + delta, end_time) + partitions.append(TimeRangePartition(current, next_time)) + current = next_time + timedelta(microseconds=1) # No overlap + + return partitions if partitions else [TimeRangePartition(start_time, end_time)] + + def commit(self, end): + """Called when batch is successfully processed.""" + # Spark handles checkpointing - usually no action needed + pass + + def read(self, partition): + """Read data for partition time range.""" + response = self._query_api( + start=partition.start_time, + end=partition.end_time + ) + + for item in response: + yield self._convert_to_row(item) +``` + +## Watermarking Support + +Support for event-time watermarking: + +```python +class WatermarkedStreamReader(DataSourceStreamReader): + def __init__(self, options, schema): + super().__init__(options, schema) + + # Watermark configuration + self.watermark_column = options.get("watermark_column") + self.watermark_delay = options.get("watermark_delay", "10 minutes") + + def read(self, partition): + """Read with event-time watermarking.""" + from datetime import datetime + + response = self._query_api( + start=partition.start_time, + end=partition.end_time + ) + + for item in response: + row = self._convert_to_row(item) + + # Validate watermark column exists + if self.watermark_column: + if not hasattr(row, self.watermark_column): + raise ValueError( + f"Watermark column '{self.watermark_column}' not found in row" + ) + + # Ensure watermark column is timestamp + watermark_value = getattr(row, self.watermark_column) + if not isinstance(watermark_value, datetime): + raise ValueError( + f"Watermark column must be timestamp, got {type(watermark_value)}" + ) + + yield row +``` + +## Stateful Streaming + +Track state across batches: + +```python +class StatefulStreamReader(DataSourceStreamReader): + def __init__(self, options, schema): + super().__init__(options, schema) + + # State management + self.checkpoint_location = options.get("checkpoint_location") + self._state = {} + + def _load_state(self): + """Load state from checkpoint location.""" + import json + import os + + if not self.checkpoint_location: + return {} + + state_file = os.path.join(self.checkpoint_location, "reader_state.json") + + if os.path.exists(state_file): + with open(state_file, 'r') as f: + return json.load(f) + + return {} + + def _save_state(self): + """Save state to checkpoint location.""" + import json + import os + + if not self.checkpoint_location: + return + + os.makedirs(self.checkpoint_location, exist_ok=True) + state_file = os.path.join(self.checkpoint_location, "reader_state.json") + + with open(state_file, 'w') as f: + json.dump(self._state, f) + + def initialOffset(self): + """Load state and return initial offset.""" + self._state = self._load_state() + + # Check if we have previous state + if "last_offset" in self._state: + return self._state["last_offset"] + + # First run - use configured start time + return self._create_initial_offset() + + def commit(self, end): + """Save state after successful batch.""" + self._state["last_offset"] = end + self._state["last_commit_time"] = datetime.now().isoformat() + self._save_state() +``` + +## Exactly-Once Semantics + +Ensure exactly-once delivery with idempotent writes: + +```python +class ExactlyOnceWriter(DataSourceStreamWriter): + def __init__(self, options, schema): + super().__init__(options, schema) + self.enable_idempotency = options.get("enable_idempotency", "true").lower() == "true" + + def write(self, iterator): + """Write with idempotency key.""" + import hashlib + from pyspark import TaskContext + + context = TaskContext.get() + partition_id = context.partitionId() + batch_id = getattr(context, 'batchId', lambda: 0)() + + for row in iterator: + # Generate idempotency key from batch_id + partition_id + row content + row_dict = row.asDict() + + if self.enable_idempotency: + idempotency_key = self._generate_idempotency_key( + batch_id, + partition_id, + row_dict + ) + row_dict["_idempotency_key"] = idempotency_key + + # Write with idempotency check + self._write_with_idempotency_check(row_dict) + + def _generate_idempotency_key(self, batch_id, partition_id, row_dict): + """Generate deterministic idempotency key.""" + import hashlib + import json + + key_data = { + "batch_id": batch_id, + "partition_id": partition_id, + "row": row_dict + } + + key_str = json.dumps(key_data, sort_keys=True) + return hashlib.sha256(key_str.encode()).hexdigest() + + def _write_with_idempotency_check(self, row_dict): + """Write only if idempotency key not seen before.""" + idempotency_key = row_dict.get("_idempotency_key") + + if idempotency_key: + # Check if already written (implementation depends on target system) + if self._is_already_written(idempotency_key): + return # Skip duplicate + + # Write data + self._write_data(row_dict) + + def commit(self, messages, batchId): + """Commit batch after all writes succeed.""" + # Log successful batch + print(f"Batch {batchId} committed successfully") + + def abort(self, messages, batchId): + """Handle failed batch.""" + # Log failed batch + print(f"Batch {batchId} aborted") +``` + +## Monitoring and Progress + +Track streaming progress: + +```python +class MonitoredStreamReader(DataSourceStreamReader): + def read(self, partition): + """Read with progress tracking.""" + from datetime import datetime + + start_time = datetime.now() + row_count = 0 + + for row in self._read_partition(partition): + row_count += 1 + yield row + + duration = (datetime.now() - start_time).total_seconds() + + # Log metrics + self._log_partition_metrics( + partition_id=partition.partition_id, + row_count=row_count, + duration=duration + ) + + def _log_partition_metrics(self, partition_id, row_count, duration): + """Log partition processing metrics.""" + print(f"Partition {partition_id}: {row_count} rows in {duration:.2f}s") +``` + +## Best Practices + +1. **Non-Overlapping Partitions**: Use microsecond adjustments to prevent duplicates +2. **Idempotency**: Generate deterministic keys for exactly-once semantics +3. **State Management**: Store offsets in Spark checkpoints +4. **Watermarking**: Support event-time processing for late data +5. **Monitoring**: Track batch progress and lag metrics +6. **Error Handling**: Implement retry logic for transient failures +7. **Backpressure**: Respect rate limits with appropriate partition sizing diff --git a/.claude/skills/spark-python-data-source/references/testing-patterns.md b/.claude/skills/spark-python-data-source/references/testing-patterns.md new file mode 100644 index 00000000..96e2e289 --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/testing-patterns.md @@ -0,0 +1,439 @@ +# Testing Patterns + +Unit and integration testing strategies for Spark data sources. + +## Basic Unit Tests + +Test data source registration and initialization: + +```python +import pytest +from pyspark.sql import SparkSession + +@pytest.fixture(scope="session") +def spark(): + """Create Spark session for tests.""" + return SparkSession.builder \ + .master("local[2]") \ + .appName("test") \ + .config("spark.sql.shuffle.partitions", "2") \ + .getOrCreate() + +def test_data_source_name(): + """Test data source name registration.""" + assert YourDataSource.name() == "your-format" + +def test_data_source_initialization(): + """Test data source can be initialized.""" + options = {"url": "http://api.example.com"} + ds = YourDataSource(options) + assert ds.options == options + +def test_missing_required_option(): + """Test error on missing required option.""" + options = {} # Missing required 'url' + + with pytest.raises(AssertionError, match="url is required"): + YourDataSource(options) +``` + +## Mocking HTTP Requests + +Test writers without external dependencies: + +```python +from unittest.mock import patch, Mock +import pytest + +@pytest.fixture +def basic_options(): + """Common options for tests.""" + return { + "url": "http://api.example.com", + "batch_size": "10" + } + +@pytest.fixture +def sample_schema(): + """Sample schema for tests.""" + from pyspark.sql.types import StructType, StructField, IntegerType, StringType + return StructType([ + StructField("id", IntegerType(), False), + StructField("name", StringType(), True) + ]) + +def test_writer_sends_batch(spark, basic_options, sample_schema): + """Test writer sends data in batches.""" + with patch('requests.post') as mock_post: + mock_post.return_value = Mock(status_code=200) + + # Create test data + df = spark.createDataFrame([ + (1, "Alice"), + (2, "Bob"), + (3, "Charlie") + ], ["id", "name"]) + + # Write using data source + df.write.format("your-format").options(**basic_options).save() + + # Verify API was called + assert mock_post.called + assert mock_post.call_count > 0 + +def test_writer_respects_batch_size(spark, basic_options, sample_schema): + """Test writer respects configured batch size.""" + with patch('requests.post') as mock_post: + mock_post.return_value = Mock(status_code=200) + + # Create 25 rows with batch_size=10 + rows = [(i, f"name_{i}") for i in range(25)] + df = spark.createDataFrame(rows, ["id", "name"]) + + df.write.format("your-format").options(**basic_options).save() + + # Should make 3 calls: 10 + 10 + 5 + assert mock_post.call_count == 3 +``` + +## Testing Readers + +Mock external API responses: + +```python +def test_reader_fetches_data(spark, basic_options): + """Test reader fetches and converts data.""" + with patch('requests.get') as mock_get: + # Mock API response + mock_response = Mock() + mock_response.json.return_value = [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"} + ] + mock_get.return_value = mock_response + + # Read using data source + df = spark.read.format("your-format").options(**basic_options).load() + + # Verify data + rows = df.collect() + assert len(rows) == 2 + assert rows[0]["id"] == 1 + assert rows[0]["name"] == "Alice" + +def test_reader_handles_empty_response(spark, basic_options): + """Test reader handles empty response.""" + with patch('requests.get') as mock_get: + mock_response = Mock() + mock_response.json.return_value = [] + mock_get.return_value = mock_response + + df = spark.read.format("your-format").options(**basic_options).load() + + assert df.count() == 0 +``` + +## Testing Partitioning + +Test partition creation logic: + +```python +def test_partitions_created(basic_options, sample_schema): + """Test correct number of partitions created.""" + options = {**basic_options, "num_partitions": "4"} + + reader = YourBatchReader(options, sample_schema) + partitions = reader.partitions() + + assert len(partitions) == 4 + +def test_partition_ranges_non_overlapping(): + """Test partitions have non-overlapping ranges.""" + from datetime import datetime, timedelta + + reader = TimeBasedReader(options, schema) + partitions = reader.partitions() + + # Check no gaps or overlaps + for i in range(len(partitions) - 1): + current_end = partitions[i].end_time + next_start = partitions[i + 1].start_time + + # Next partition should start right after current ends + assert next_start >= current_end +``` + +## Testing Streaming + +Test offset management and streaming logic: + +```python +def test_initial_offset(): + """Test initial offset is correct.""" + from datetime import datetime + + reader = YourStreamReader(options, schema) + initial = reader.initialOffset() + + # Should be valid JSON + import json + offset_dict = json.loads(initial) + + assert "timestamp" in offset_dict + +def test_latest_offset_advances(): + """Test latest offset advances over time.""" + reader = YourStreamReader(options, schema) + + offset1 = reader.latestOffset() + import time + time.sleep(0.1) + offset2 = reader.latestOffset() + + # Offset should advance + assert offset2 > offset1 or offset2 != offset1 + +def test_partitions_non_overlapping(basic_options, sample_schema): + """Test streaming partitions don't overlap.""" + reader = YourStreamReader(basic_options, sample_schema) + + start = reader.initialOffset() + end = reader.latestOffset() + + partitions = reader.partitions(start, end) + + # Verify no overlaps + for i in range(len(partitions) - 1): + assert partitions[i].end_time < partitions[i + 1].start_time +``` + +## Testing Type Conversion + +Test type mapping and conversion: + +```python +def test_convert_timestamp(): + """Test timestamp conversion.""" + from datetime import datetime + from pyspark.sql.types import TimestampType + + dt = datetime(2024, 1, 1, 12, 0, 0) + result = convert_external_to_spark(dt, TimestampType()) + + assert isinstance(result, datetime) + assert result == dt + +def test_convert_null_values(): + """Test null value handling.""" + from pyspark.sql.types import StringType + + result = convert_external_to_spark(None, StringType()) + assert result is None + +def test_convert_invalid_type(): + """Test error on invalid type conversion.""" + from pyspark.sql.types import IntegerType + + with pytest.raises(ValueError, match="Cannot convert"): + convert_external_to_spark("not_a_number", IntegerType()) +``` + +## Integration Tests with Testcontainers + +Run end-to-end tests against real systems: + +```python +import pytest +from testcontainers.postgres import PostgresContainer + +@pytest.fixture(scope="session") +def postgres_container(): + """Start PostgreSQL container for integration tests.""" + with PostgresContainer("postgres:15") as container: + yield container + +@pytest.fixture +def postgres_connection(postgres_container): + """Create connection to test database.""" + import psycopg2 + + conn = psycopg2.connect(postgres_container.get_connection_url()) + cursor = conn.cursor() + + # Create test table + cursor.execute(""" + CREATE TABLE test_data ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + value INTEGER + ) + """) + conn.commit() + + yield conn + + conn.close() + +def test_write_integration(spark, postgres_container, postgres_connection): + """Integration test for writing to PostgreSQL.""" + # Create test data + df = spark.createDataFrame([ + (1, "Alice", 100), + (2, "Bob", 200) + ], ["id", "name", "value"]) + + # Write using data source + df.write.format("your-format") \ + .option("url", postgres_container.get_connection_url()) \ + .option("table", "test_data") \ + .save() + + # Verify data written + cursor = postgres_connection.cursor() + cursor.execute("SELECT COUNT(*) FROM test_data") + count = cursor.fetchone()[0] + + assert count == 2 + +def test_read_integration(spark, postgres_container, postgres_connection): + """Integration test for reading from PostgreSQL.""" + # Insert test data + cursor = postgres_connection.cursor() + cursor.execute("INSERT INTO test_data (name, value) VALUES ('Alice', 100)") + cursor.execute("INSERT INTO test_data (name, value) VALUES ('Bob', 200)") + postgres_connection.commit() + + # Read using data source + df = spark.read.format("your-format") \ + .option("url", postgres_container.get_connection_url()) \ + .option("table", "test_data") \ + .load() + + # Verify data + assert df.count() == 2 + names = [row["name"] for row in df.collect()] + assert "Alice" in names + assert "Bob" in names +``` + +## Performance Tests + +Test performance characteristics: + +```python +import time + +def test_write_performance(spark, basic_options): + """Test write performance meets requirements.""" + # Create large dataset + rows = [(i, f"name_{i}") for i in range(10000)] + df = spark.createDataFrame(rows, ["id", "name"]) + + start = time.time() + df.write.format("your-format").options(**basic_options).save() + duration = time.time() - start + + # Should complete in reasonable time + assert duration < 30.0 # 30 seconds + + # Calculate throughput + throughput = len(rows) / duration + print(f"Write throughput: {throughput:.0f} rows/second") + +def test_partition_read_parallelism(spark, basic_options): + """Test reads execute in parallel.""" + options = {**basic_options, "num_partitions": "4"} + + df = spark.read.format("your-format").options(**options).load() + + # Check partition count + assert df.rdd.getNumPartitions() == 4 +``` + +## Test Fixtures and Utilities + +Reusable test fixtures: + +```python +import pytest +from pyspark.sql import SparkSession + +@pytest.fixture(scope="session") +def spark(): + """Shared Spark session.""" + return SparkSession.builder \ + .master("local[2]") \ + .appName("test") \ + .config("spark.sql.shuffle.partitions", "2") \ + .getOrCreate() + +@pytest.fixture +def sample_dataframe(spark): + """Sample DataFrame for testing.""" + return spark.createDataFrame([ + (1, "Alice", 25), + (2, "Bob", 30), + (3, "Charlie", 35) + ], ["id", "name", "age"]) + +@pytest.fixture +def temp_output_path(tmp_path): + """Temporary output path.""" + return str(tmp_path / "output") + +def assert_dataframes_equal(df1, df2): + """Assert two DataFrames are equal.""" + assert df1.schema == df2.schema + assert df1.count() == df2.count() + + rows1 = sorted(df1.collect()) + rows2 = sorted(df2.collect()) + + assert rows1 == rows2 +``` + +## Test Organization + +Structure tests by functionality: + +``` +tests/ +├── unit/ +│ ├── test_datasource.py # DataSource class tests +│ ├── test_reader.py # Reader tests +│ ├── test_writer.py # Writer tests +│ ├── test_partitioning.py # Partitioning logic +│ └── test_type_conversion.py # Type conversion +├── integration/ +│ ├── test_read_integration.py # End-to-end read tests +│ ├── test_write_integration.py # End-to-end write tests +│ └── test_streaming.py # Streaming tests +├── performance/ +│ └── test_performance.py # Performance tests +└── conftest.py # Shared fixtures +``` + +## Running Tests + +```bash +# Run all tests +poetry run pytest + +# Run specific test file +poetry run pytest tests/unit/test_writer.py + +# Run specific test +poetry run pytest tests/unit/test_writer.py::test_writer_sends_batch + +# Run with coverage +poetry run pytest --cov=your_package --cov-report=html + +# Run only unit tests +poetry run pytest tests/unit/ + +# Run with verbose output +poetry run pytest -v + +# Run with print statements +poetry run pytest -s +``` diff --git a/.claude/skills/spark-python-data-source/references/type-conversion.md b/.claude/skills/spark-python-data-source/references/type-conversion.md new file mode 100644 index 00000000..a55f0795 --- /dev/null +++ b/.claude/skills/spark-python-data-source/references/type-conversion.md @@ -0,0 +1,370 @@ +# Type Conversion + +Bidirectional mapping between Spark types and external system types. + +## Spark to External System + +Convert Spark/Python values to external system types: + +```python +def convert_spark_to_external(value, external_type): + """Convert Spark/Python value to external system type.""" + if value is None: + return None + + external_type_lower = external_type.lower() + + # UUID conversion + if "uuid" in external_type_lower: + import uuid + if isinstance(value, uuid.UUID): + return value + return uuid.UUID(str(value)) + + # Timestamp conversion + if "timestamp" in external_type_lower: + from datetime import datetime + if isinstance(value, datetime): + return value + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value) + + # IP address conversion + if "inet" in external_type_lower: + import ipaddress + if isinstance(value, (ipaddress.IPv4Address, ipaddress.IPv6Address)): + return value + return ipaddress.ip_address(str(value)) + + # Decimal conversion + if "decimal" in external_type_lower: + from decimal import Decimal + if isinstance(value, Decimal): + return value + return Decimal(str(value)) + + # Collections + if "list" in external_type_lower or "set" in external_type_lower: + if not isinstance(value, (list, set)): + raise ValueError(f"Expected list/set, got {type(value)}") + return list(value) + + if "map" in external_type_lower: + if not isinstance(value, dict): + raise ValueError(f"Expected dict, got {type(value)}") + return value + + # Numeric types + if "int" in external_type_lower: + return int(value) + if "float" in external_type_lower or "double" in external_type_lower: + return float(value) + + # Boolean + if "bool" in external_type_lower: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() in ("true", "1", "yes") + return bool(value) + + # Default: return as-is + return value +``` + +## External System to Spark + +Convert external values to Spark types: + +```python +def convert_external_to_spark(value, spark_type): + """Convert external system value to Spark type.""" + from pyspark.sql.types import ( + StringType, IntegerType, LongType, FloatType, DoubleType, + BooleanType, TimestampType, DateType + ) + from datetime import datetime, date + + if value is None: + return None + + try: + if isinstance(spark_type, StringType): + return str(value) + + elif isinstance(spark_type, BooleanType): + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() in ("true", "1", "yes") + return bool(value) + + elif isinstance(spark_type, (IntegerType, LongType)): + if isinstance(value, bool): + raise ValueError("Cannot convert boolean to integer") + return int(value) + + elif isinstance(spark_type, (FloatType, DoubleType)): + if isinstance(value, bool): + raise ValueError("Cannot convert boolean to float") + return float(value) + + elif isinstance(spark_type, TimestampType): + if isinstance(value, datetime): + return value + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + raise ValueError(f"Cannot convert {type(value)} to timestamp") + + elif isinstance(spark_type, DateType): + if isinstance(value, date) and not isinstance(value, datetime): + return value + if isinstance(value, datetime): + return value.date() + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")).date() + raise ValueError(f"Cannot convert {type(value)} to date") + + else: + return value + + except (ValueError, TypeError) as e: + raise ValueError( + f"Failed to convert '{value}' (type: {type(value).__name__}) " + f"to {spark_type}: {e}" + ) +``` + +## Cassandra-Specific Types + +Handle Cassandra complex types: + +```python +def convert_cassandra_to_spark(value): + """Handle Cassandra-specific complex types.""" + if value is None: + return None + + from cassandra.util import ( + Date, Time, Duration, OrderedMap, SortedSet, + Point, LineString, Polygon + ) + import uuid + + # Cassandra Date to Python date + if isinstance(value, Date): + return value.date() + + # Cassandra Time to nanoseconds (LongType) + if isinstance(value, Time): + return value.nanosecond + + # UUID to string + if isinstance(value, uuid.UUID): + return str(value) + + # Duration to structured dict + if isinstance(value, Duration): + return { + "months": value.months, + "days": value.days, + "nanoseconds": value.nanoseconds + } + + # OrderedMap to dict + if isinstance(value, OrderedMap): + return dict(value) + + # SortedSet to list + if isinstance(value, SortedSet): + return list(value) + + # Geospatial types to WKT string + if isinstance(value, (Point, LineString, Polygon)): + return str(value) + + return value +``` + +## Schema Inference + +Infer Spark types from Python values: + +```python +def infer_spark_type(value): + """Infer Spark type from Python value.""" + from pyspark.sql.types import ( + StringType, IntegerType, LongType, FloatType, DoubleType, + BooleanType, TimestampType, DateType + ) + from datetime import datetime, date + + if value is None: + return StringType() + + # Check bool before int (bool is subclass of int) + if isinstance(value, bool): + return BooleanType() + + if isinstance(value, int): + return LongType() + + if isinstance(value, float): + return DoubleType() + + if isinstance(value, datetime): + return TimestampType() + + if isinstance(value, date): + return DateType() + + # Default to string + return StringType() +``` + +## External Type to Spark Type Mapping + +Map external system types to Spark types: + +```python +def map_external_type_to_spark(external_type): + """Map external system types to Spark types.""" + from pyspark.sql.types import ( + StringType, IntegerType, LongType, FloatType, DoubleType, + BooleanType, TimestampType, DateType, BinaryType + ) + + type_str = str(external_type).lower() + + # String types + if any(t in type_str for t in ["varchar", "text", "char", "string", "uuid"]): + return StringType() + + # Integer types + if "int" in type_str and "big" not in type_str: + return IntegerType() + if "bigint" in type_str or "long" in type_str: + return LongType() + + # Floating point + if "float" in type_str: + return FloatType() + if "double" in type_str or "decimal" in type_str: + return DoubleType() + + # Boolean + if "bool" in type_str: + return BooleanType() + + # Temporal types + if "timestamp" in type_str: + return TimestampType() + if "date" in type_str: + return DateType() + + # Binary + if "blob" in type_str or "binary" in type_str: + return BinaryType() + + # Default fallback + return StringType() +``` + +## JSON Encoding + +Handle datetime serialization for JSON APIs: + +```python +import json +from datetime import date, datetime +from decimal import Decimal + +class ExtendedJsonEncoder(json.JSONEncoder): + """JSON encoder that handles datetime, date, and Decimal.""" + + def default(self, o): + if isinstance(o, (datetime, date)): + return o.isoformat() + + if isinstance(o, Decimal): + return float(o) + + return super().default(o) + +# Usage +def send_as_json(data): + import requests + + payload = json.dumps(data, cls=ExtendedJsonEncoder) + requests.post(url, data=payload, headers={"Content-Type": "application/json"}) +``` + +## Complete Row Conversion + +Convert entire rows with schema: + +```python +def convert_row_to_external(row, column_types): + """Convert entire Spark row to external system format.""" + row_dict = row.asDict() if hasattr(row, "asDict") else dict(row) + + converted = {} + for col, value in row_dict.items(): + external_type = column_types.get(col, "text") + converted[col] = convert_spark_to_external(value, external_type) + + return converted + +def convert_external_to_row(data, schema): + """Convert external data to Spark Row.""" + from pyspark.sql import Row + + # Create mapping of column names to types + schema_map = {field.name: field.dataType for field in schema.fields} + + row_dict = {} + for col, value in data.items(): + if col in schema_map: + spark_type = schema_map[col] + row_dict[col] = convert_external_to_spark(value, spark_type) + + # Add None for missing columns + for field in schema.fields: + if field.name not in row_dict: + row_dict[field.name] = None + + return Row(**row_dict) +``` + +## Validation + +Validate type conversions: + +```python +def validate_conversion(value, expected_type): + """Validate that value matches expected type after conversion.""" + type_checks = { + "int": lambda v: isinstance(v, int) and not isinstance(v, bool), + "long": lambda v: isinstance(v, int) and not isinstance(v, bool), + "float": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool), + "double": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool), + "string": lambda v: isinstance(v, str), + "boolean": lambda v: isinstance(v, bool), + "timestamp": lambda v: isinstance(v, datetime), + "date": lambda v: isinstance(v, date) and not isinstance(v, datetime), + } + + expected_type_lower = expected_type.lower() + for type_name, check in type_checks.items(): + if type_name in expected_type_lower: + if not check(value): + raise ValueError( + f"Value {value} (type: {type(value)}) does not match " + f"expected type {expected_type}" + ) + return + + # No specific check - accept any value +``` diff --git a/app.yaml b/app.yaml index 288b55e9..c8e33371 100644 --- a/app.yaml +++ b/app.yaml @@ -1,7 +1,7 @@ command: - bash - -c - - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_databricks.py && python app.py" + - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && python app.py" env: - name: HOME value: /app/python/source_code @@ -10,4 +10,9 @@ env: - name: DATABRICKS_TOKEN valueFrom: DATABRICKS_TOKEN - name: ANTHROPIC_MODEL - value: databricks-claude-opus-4-6 \ No newline at end of file + value: databricks-claude-opus-4-6 + - name: GEMINI_MODEL + value: databricks-gemini-3-1-pro + #OPTIONAL: Move to the new Databricks Gateway if you have access (recommended), otherwise it will default to the older endpoint + - name: DATABRICKS_GATEWAY_HOST + value: https://6051921418418893.ai-gateway.staging.cloud.databricks.com \ No newline at end of file diff --git a/setup_claude.py b/setup_claude.py index 2341705f..b12bfceb 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -14,10 +14,25 @@ claude_dir.mkdir(exist_ok=True) # 1. Write settings.json for Databricks model serving +# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST +gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") +databricks_host = os.environ.get("DATABRICKS_HOST", "").rstrip("/") +base_host = gateway_host if gateway_host else databricks_host + +if gateway_host: + print(f"Using Databricks AI Gateway: {gateway_host}") +else: + print(f"Using Databricks Host: {databricks_host}") + +if gateway_host: + anthropic_base_url = f"{gateway_host}/anthropic" +else: + anthropic_base_url = f"{databricks_host}/serving-endpoints/anthropic" + settings = { "env": { - "ANTHROPIC_MODEL": os.environ.get("ANTHROPIC_MODEL", "databricks-claude-sonnet-4-5"), - "ANTHROPIC_BASE_URL": f"{os.environ['DATABRICKS_HOST']}/serving-endpoints/anthropic", + "ANTHROPIC_MODEL": os.environ.get("ANTHROPIC_MODEL", "databricks-claude-sonnet-4-6"), + "ANTHROPIC_BASE_URL": anthropic_base_url, "ANTHROPIC_AUTH_TOKEN": os.environ["DATABRICKS_TOKEN"], "ANTHROPIC_CUSTOM_HEADERS": "x-databricks-use-coding-agent-mode: true" } diff --git a/setup_gemini.py b/setup_gemini.py new file mode 100644 index 00000000..3273fb66 --- /dev/null +++ b/setup_gemini.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +"""Configure Gemini CLI with Databricks Model Serving. + +Gemini CLI uses the Google Generative Language API protocol, not OpenAI-compatible. +Databricks provides a Google-native endpoint at /serving-endpoints/google +(similar to /serving-endpoints/anthropic for Claude). + +PR #11893 (by Databricks engineer AarushiShah) added auto-detection of *.databricks.com +URLs, switching to Bearer token auth automatically. + +Auth: GEMINI_API_KEY_AUTH_MECHANISM=bearer sends Databricks PAT as Bearer token. +""" +import os +import json +import subprocess +from pathlib import Path + +# Set HOME if not properly set +if not os.environ.get("HOME") or os.environ["HOME"] == "/": + os.environ["HOME"] = "/app/python/source_code" + +home = Path(os.environ["HOME"]) + +host = os.environ.get("DATABRICKS_HOST", "") +token = os.environ.get("DATABRICKS_TOKEN", "") +gemini_model = os.environ.get("GEMINI_MODEL", "databricks-gemini-2-5-flash") + +if not host or not token: + print("Warning: DATABRICKS_HOST or DATABRICKS_TOKEN not set, skipping Gemini CLI config") + exit(0) + +# Strip trailing slash from host +host = host.rstrip("/") + +# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST +gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") +if gateway_host: + gemini_base_url = f"{gateway_host}/gemini" + print(f"Using Databricks AI Gateway: {gateway_host}") +else: + gemini_base_url = f"{host}/serving-endpoints/google" + print(f"Using Databricks Host: {host}") + +# 1. Install Gemini CLI into ~/.local/bin (same approach as Claude Code) +local_bin = home / ".local" / "bin" +local_bin.mkdir(parents=True, exist_ok=True) +gemini_bin = local_bin / "gemini" + +if not gemini_bin.exists(): + print("Installing Gemini CLI...") + # Use --prefix ~/.local so npm installs directly into ~/.local/bin (avoids EACCES on /usr/local) + npm_prefix = str(home / ".local") + result = subprocess.run( + ["npm", "install", "-g", f"--prefix={npm_prefix}", "@google/gemini-cli"], + capture_output=True, text=True, + env={**os.environ, "HOME": str(home)} + ) + if result.returncode == 0: + print(f"Gemini CLI installed to {gemini_bin}") + else: + print(f"Gemini CLI install warning: {result.stderr}") +else: + print(f"Gemini CLI already installed at {gemini_bin}") + +# 2. Create ~/.gemini directory and configure environment +gemini_dir = home / ".gemini" +gemini_dir.mkdir(exist_ok=True) + +# Write .env file with Databricks endpoint configuration +# Gemini CLI auto-loads env from ~/.gemini/.env +# The Google-native endpoint on Databricks mirrors /serving-endpoints/anthropic +env_content = f"""# Databricks Model Serving - Google Gemini native endpoint +GEMINI_MODEL={gemini_model} +GOOGLE_GEMINI_BASE_URL={gemini_base_url} +GEMINI_API_KEY_AUTH_MECHANISM="bearer" +GEMINI_API_KEY={token} +""" + +env_path = gemini_dir / ".env" +env_path.write_text(env_content) +env_path.chmod(0o600) +print(f"Gemini CLI env configured: {env_path}") + +# 3. Write settings.json with model preferences +settings = { + "theme": "Default", + "selectedAuthType": "api-key" +} + +settings_path = gemini_dir / "settings.json" +settings_path.write_text(json.dumps(settings, indent=2)) +print(f"Gemini CLI settings configured: {settings_path}") + +print("\nGemini CLI ready! Usage:") +print(" gemini # Start Gemini CLI") +print(f" gemini -m gemini-2.5-flash # Use Gemini 2.5 Flash") +print(f" gemini -m gemini-2.5-pro # Use Gemini 2.5 Pro") +print(f"\nEndpoint: {gemini_base_url}") +print("Auth: Bearer token (Databricks PAT)") diff --git a/setup_opencode.py b/setup_opencode.py new file mode 100644 index 00000000..bfcc7900 --- /dev/null +++ b/setup_opencode.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +"""Configure OpenCode CLI with Databricks Model Serving as an OpenAI-compatible provider.""" +import os +import json +import subprocess +from pathlib import Path + +# Set HOME if not properly set +if not os.environ.get("HOME") or os.environ["HOME"] == "/": + os.environ["HOME"] = "/app/python/source_code" + +home = Path(os.environ["HOME"]) + +host = os.environ.get("DATABRICKS_HOST", "") +token = os.environ.get("DATABRICKS_TOKEN", "") +anthropic_model = os.environ.get("ANTHROPIC_MODEL", "databricks-claude-sonnet-4-6") + +if not host or not token: + print("Warning: DATABRICKS_HOST or DATABRICKS_TOKEN not set, skipping OpenCode config") + exit(0) + +# Strip trailing slash from host +host = host.rstrip("/") + +# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST +gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") +if gateway_host: + print(f"Using Databricks AI Gateway: {gateway_host}") +else: + print(f"Using Databricks Host: {host}") + +# 1. Install OpenCode CLI into ~/.local/bin (same approach as Claude Code) +local_bin = home / ".local" / "bin" +local_bin.mkdir(parents=True, exist_ok=True) +opencode_bin = local_bin / "opencode" + +if not opencode_bin.exists(): + print("Installing OpenCode CLI...") + # Use --prefix ~/.local so npm installs directly into ~/.local/bin (avoids EACCES on /usr/local) + npm_prefix = str(home / ".local") + result = subprocess.run( + ["npm", "install", "-g", f"--prefix={npm_prefix}", "opencode-ai@latest"], + capture_output=True, text=True, + env={**os.environ, "HOME": str(home)} + ) + if result.returncode == 0: + print(f"OpenCode CLI installed to {opencode_bin}") + else: + print(f"OpenCode install warning: {result.stderr}") +else: + print(f"OpenCode CLI already installed at {opencode_bin}") + +# 2. Write global opencode.json config +# OpenCode looks for config at ~/.config/opencode/opencode.json (global) +# and ./opencode.json (project-level) +opencode_config_dir = home / ".config" / "opencode" +opencode_config_dir.mkdir(parents=True, exist_ok=True) + +if gateway_host: + # Gateway mode: separate providers for different API protocols + # - Anthropic/Gemini models use MLflow endpoint: {gateway}/mlflow/v1/chat/completions + # - OpenAI/GPT models use OpenAI endpoint: {gateway}/openai/v1/responses + opencode_config = { + "$schema": "https://opencode.ai/config.json", + "provider": { + "databricks": { + "npm": "@ai-sdk/openai-compatible", + "name": "Databricks AI Gateway (MLflow)", + "options": { + "baseURL": f"{gateway_host}/mlflow/v1", + "apiKey": "{env:DATABRICKS_TOKEN}" + }, + "models": { + "databricks-claude-opus-4-6": { + "name": "Claude Opus 4.6 (Databricks)", + "limit": { + "context": 200000, + "output": 16384 + } + }, + "databricks-claude-sonnet-4-6": { + "name": "Claude Sonnet 4.6 (Databricks)", + "limit": { + "context": 200000, + "output": 8192 + } + }, + "databricks-gemini-2-5-flash": { + "name": "Gemini 2.5 Flash (Databricks)", + "limit": { + "context": 1000000, + "output": 8192 + } + }, + "databricks-gemini-2-5-pro": { + "name": "Gemini 2.5 Pro (Databricks)", + "limit": { + "context": 1000000, + "output": 8192 + } + }, + "databricks-gemini-3-1-pro": { + "name": "Gemini 3.1 Pro (Databricks)", + "limit": { + "context": 1000000, + "output": 8192 + } + }, + } + }, + "databricks-openai": { + "npm": "@ai-sdk/openai-compatible", + "name": "Databricks AI Gateway (OpenAI)", + "options": { + "baseURL": f"{gateway_host}/openai/v1", + "apiKey": "{env:DATABRICKS_TOKEN}" + }, + "models": { + "databricks-gpt-5-2-codex": { + "name": "GPT 5.2 Codex (Databricks)", + "limit": { + "context": 200000, + "output": 16384 + } + }, + "databricks-gpt-5-1-codex-max": { + "name": "GPT 5.1 Codex Max (Databricks)", + "limit": { + "context": 200000, + "output": 16384 + } + } + } + } + }, + "model": f"databricks/{anthropic_model}" + } +else: + # Fallback: single provider using DATABRICKS_HOST /serving-endpoints (OpenAI-compatible) + opencode_config = { + "$schema": "https://opencode.ai/config.json", + "provider": { + "databricks": { + "npm": "@ai-sdk/openai-compatible", + "name": "Databricks Model Serving", + "options": { + "baseURL": f"{host}/serving-endpoints", + "apiKey": "{env:DATABRICKS_TOKEN}" + }, + "models": { + "databricks-claude-opus-4-6": { + "name": "Claude Opus 4.6 (Databricks)", + "limit": { + "context": 200000, + "output": 16384 + } + }, + "databricks-claude-sonnet-4-6": { + "name": "Claude Sonnet 4.6 (Databricks)", + "limit": { + "context": 200000, + "output": 8192 + } + }, + "databricks-gemini-2-5-flash": { + "name": "Gemini 2.5 Flash (Databricks)", + "limit": { + "context": 1000000, + "output": 8192 + } + }, + "databricks-gemini-2-5-pro": { + "name": "Gemini 2.5 Pro (Databricks)", + "limit": { + "context": 1000000, + "output": 8192 + } + }, + "databricks-gemini-3-1-pro": { + "name": "Gemini 3.1 Pro (Databricks)", + "limit": { + "context": 1000000, + "output": 8192 + } + }, + } + } + }, + "model": f"databricks/{anthropic_model}" + } + +config_path = opencode_config_dir / "opencode.json" +config_path.write_text(json.dumps(opencode_config, indent=2)) +print(f"OpenCode configured: {config_path}") + +# 3. Also create auth credentials for the databricks provider(s) +# OpenCode stores credentials at ~/.local/share/opencode/auth.json +opencode_data_dir = home / ".local" / "share" / "opencode" +opencode_data_dir.mkdir(parents=True, exist_ok=True) + +auth_data = { + "databricks": { + "api_key": token + } +} +if gateway_host: + auth_data["databricks-openai"] = { + "api_key": token + } + +auth_path = opencode_data_dir / "auth.json" +auth_path.write_text(json.dumps(auth_data, indent=2)) +auth_path.chmod(0o600) +print(f"OpenCode auth configured: {auth_path}") + +print(f"\nOpenCode ready! Default model: {anthropic_model}") +print(" opencode # Start OpenCode TUI") +if gateway_host: + print(" opencode -m databricks-openai/databricks-gpt-5-2-codex # Use GPT 5.2 Codex") +print(" opencode -m databricks/databricks-gemini-2-5-flash # Use Gemini") +print(f" opencode -m databricks/{anthropic_model} # Use Claude (default)") From ef41ce05c92d9da93492ea10f65dcb07be6f8fb7 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 08:34:31 -0500 Subject: [PATCH 005/382] feat: use DATABRICKS_GATEWAY_TOKEN for auth and fix OpenCode baseURLs - All 3 setup scripts: use DATABRICKS_GATEWAY_TOKEN when gateway host is present, fall back to DATABRICKS_TOKEN on current gateway - setup_opencode.py: full endpoint paths for OpenCode - Anthropic/Gemini: {gateway}/mlflow/v1/chat/completions - OpenAI/GPT: {gateway}/openai/v1/responses - app.yaml: add DATABRICKS_GATEWAY_TOKEN secret Co-Authored-By: Claude Opus 4.6 --- app.yaml | 4 +++- setup_claude.py | 14 +++++++++----- setup_gemini.py | 9 ++++++++- setup_opencode.py | 37 ++++++++++++++++++++++++------------- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/app.yaml b/app.yaml index c8e33371..41c17d19 100644 --- a/app.yaml +++ b/app.yaml @@ -15,4 +15,6 @@ env: value: databricks-gemini-3-1-pro #OPTIONAL: Move to the new Databricks Gateway if you have access (recommended), otherwise it will default to the older endpoint - name: DATABRICKS_GATEWAY_HOST - value: https://6051921418418893.ai-gateway.staging.cloud.databricks.com \ No newline at end of file + value: https://6051921418418893.ai-gateway.staging.cloud.databricks.com + - name: DATABRICKS_GATEWAY_TOKEN + valueFrom: DATABRICKS_GATEWAY_TOKEN \ No newline at end of file diff --git a/setup_claude.py b/setup_claude.py index b12bfceb..c619452f 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -17,23 +17,27 @@ # Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") databricks_host = os.environ.get("DATABRICKS_HOST", "").rstrip("/") -base_host = gateway_host if gateway_host else databricks_host if gateway_host: - print(f"Using Databricks AI Gateway: {gateway_host}") -else: - print(f"Using Databricks Host: {databricks_host}") + gateway_token = os.environ.get("DATABRICKS_GATEWAY_TOKEN", "") + if not gateway_token: + print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_GATEWAY_TOKEN missing, falling back to DATABRICKS_HOST") + gateway_host = "" if gateway_host: anthropic_base_url = f"{gateway_host}/anthropic" + auth_token = gateway_token + print(f"Using Databricks AI Gateway: {gateway_host}") else: anthropic_base_url = f"{databricks_host}/serving-endpoints/anthropic" + auth_token = os.environ["DATABRICKS_TOKEN"] + print(f"Using Databricks Host: {databricks_host}") settings = { "env": { "ANTHROPIC_MODEL": os.environ.get("ANTHROPIC_MODEL", "databricks-claude-sonnet-4-6"), "ANTHROPIC_BASE_URL": anthropic_base_url, - "ANTHROPIC_AUTH_TOKEN": os.environ["DATABRICKS_TOKEN"], + "ANTHROPIC_AUTH_TOKEN": auth_token, "ANTHROPIC_CUSTOM_HEADERS": "x-databricks-use-coding-agent-mode: true" } } diff --git a/setup_gemini.py b/setup_gemini.py index 3273fb66..d4883b50 100644 --- a/setup_gemini.py +++ b/setup_gemini.py @@ -34,11 +34,18 @@ # Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") +gateway_token = os.environ.get("DATABRICKS_GATEWAY_TOKEN", "") if gateway_host else "" +if gateway_host and not gateway_token: + print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_GATEWAY_TOKEN missing, falling back to DATABRICKS_HOST") + gateway_host = "" + if gateway_host: gemini_base_url = f"{gateway_host}/gemini" + auth_token = gateway_token print(f"Using Databricks AI Gateway: {gateway_host}") else: gemini_base_url = f"{host}/serving-endpoints/google" + auth_token = token print(f"Using Databricks Host: {host}") # 1. Install Gemini CLI into ~/.local/bin (same approach as Claude Code) @@ -73,7 +80,7 @@ GEMINI_MODEL={gemini_model} GOOGLE_GEMINI_BASE_URL={gemini_base_url} GEMINI_API_KEY_AUTH_MECHANISM="bearer" -GEMINI_API_KEY={token} +GEMINI_API_KEY={auth_token} """ env_path = gemini_dir / ".env" diff --git a/setup_opencode.py b/setup_opencode.py index bfcc7900..b9d3c603 100644 --- a/setup_opencode.py +++ b/setup_opencode.py @@ -22,8 +22,13 @@ # Strip trailing slash from host host = host.rstrip("/") -# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST +# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to current gateway (DATABRICKS_HOST) gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") +gateway_token = os.environ.get("DATABRICKS_GATEWAY_TOKEN", "") if gateway_host else "" +if gateway_host and not gateway_token: + print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_GATEWAY_TOKEN missing, falling back to DATABRICKS_HOST") + gateway_host = "" + if gateway_host: print(f"Using Databricks AI Gateway: {gateway_host}") else: @@ -67,8 +72,8 @@ "npm": "@ai-sdk/openai-compatible", "name": "Databricks AI Gateway (MLflow)", "options": { - "baseURL": f"{gateway_host}/mlflow/v1", - "apiKey": "{env:DATABRICKS_TOKEN}" + "baseURL": f"{gateway_host}/mlflow/v1/chat/completions", + "apiKey": "{env:DATABRICKS_GATEWAY_TOKEN}" }, "models": { "databricks-claude-opus-4-6": { @@ -112,8 +117,8 @@ "npm": "@ai-sdk/openai-compatible", "name": "Databricks AI Gateway (OpenAI)", "options": { - "baseURL": f"{gateway_host}/openai/v1", - "apiKey": "{env:DATABRICKS_TOKEN}" + "baseURL": f"{gateway_host}/openai/v1/responses", + "apiKey": "{env:DATABRICKS_GATEWAY_TOKEN}" }, "models": { "databricks-gpt-5-2-codex": { @@ -136,7 +141,7 @@ "model": f"databricks/{anthropic_model}" } else: - # Fallback: single provider using DATABRICKS_HOST /serving-endpoints (OpenAI-compatible) + # Fallback: current gateway using DATABRICKS_HOST /serving-endpoints (OpenAI-compatible) opencode_config = { "$schema": "https://opencode.ai/config.json", "provider": { @@ -198,14 +203,20 @@ opencode_data_dir = home / ".local" / "share" / "opencode" opencode_data_dir.mkdir(parents=True, exist_ok=True) -auth_data = { - "databricks": { - "api_key": token - } -} if gateway_host: - auth_data["databricks-openai"] = { - "api_key": token + auth_data = { + "databricks": { + "api_key": gateway_token + }, + "databricks-openai": { + "api_key": gateway_token + } + } +else: + auth_data = { + "databricks": { + "api_key": token + } } auth_path = opencode_data_dir / "auth.json" From fd3f1b6819d382a3a3382ac652608a7749f3ea63 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 12:55:05 -0500 Subject: [PATCH 006/382] chore: remove hardcoded DATABRICKS_HOST from app.yaml Platform injects DATABRICKS_HOST automatically. Gateway host is the primary auth path; fallback uses platform-injected host. Co-Authored-By: Claude Opus 4.6 --- app.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/app.yaml b/app.yaml index 41c17d19..48ed0d58 100644 --- a/app.yaml +++ b/app.yaml @@ -5,8 +5,6 @@ command: env: - name: HOME value: /app/python/source_code - - name: DATABRICKS_HOST - value: https://fevm-serverless-9cefok.cloud.databricks.com - name: DATABRICKS_TOKEN valueFrom: DATABRICKS_TOKEN - name: ANTHROPIC_MODEL From 67efcb99f862bde36016688a380fc310ee4ca823 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 13:02:29 -0500 Subject: [PATCH 007/382] fix: remove duplicate path segments from OpenCode gateway baseURLs SDK auto-appends /chat/completions and /responses to baseURL, so baseURL should be just /mlflow/v1 and /openai/v1, not the full path. Co-Authored-By: Claude Opus 4.6 --- setup_opencode.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup_opencode.py b/setup_opencode.py index b9d3c603..7b7c2469 100644 --- a/setup_opencode.py +++ b/setup_opencode.py @@ -63,8 +63,9 @@ if gateway_host: # Gateway mode: separate providers for different API protocols - # - Anthropic/Gemini models use MLflow endpoint: {gateway}/mlflow/v1/chat/completions - # - OpenAI/GPT models use OpenAI endpoint: {gateway}/openai/v1/responses + # SDK auto-appends /chat/completions and /responses to baseURL + # - Anthropic/Gemini models: baseURL={gateway}/mlflow/v1 → /mlflow/v1/chat/completions + # - OpenAI/GPT models: baseURL={gateway}/openai/v1 → /openai/v1/responses opencode_config = { "$schema": "https://opencode.ai/config.json", "provider": { @@ -72,7 +73,7 @@ "npm": "@ai-sdk/openai-compatible", "name": "Databricks AI Gateway (MLflow)", "options": { - "baseURL": f"{gateway_host}/mlflow/v1/chat/completions", + "baseURL": f"{gateway_host}/mlflow/v1", "apiKey": "{env:DATABRICKS_GATEWAY_TOKEN}" }, "models": { @@ -117,7 +118,7 @@ "npm": "@ai-sdk/openai-compatible", "name": "Databricks AI Gateway (OpenAI)", "options": { - "baseURL": f"{gateway_host}/openai/v1/responses", + "baseURL": f"{gateway_host}/openai/v1", "apiKey": "{env:DATABRICKS_GATEWAY_TOKEN}" }, "models": { From 59b15f09a1713caa7336548b42fe92152a5548e2 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 13:19:37 -0500 Subject: [PATCH 008/382] fix: use Gemini CLI nightly and correct selectedAuthType value Databricks docs specify @google/gemini-cli@nightly for gateway support. Fix selectedAuthType from "api-key" to "gemini-api-key" to auto-skip the auth prompt when GEMINI_API_KEY is set in .env. Co-Authored-By: Claude Opus 4.6 --- setup_gemini.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup_gemini.py b/setup_gemini.py index d4883b50..fc76c517 100644 --- a/setup_gemini.py +++ b/setup_gemini.py @@ -58,7 +58,7 @@ # Use --prefix ~/.local so npm installs directly into ~/.local/bin (avoids EACCES on /usr/local) npm_prefix = str(home / ".local") result = subprocess.run( - ["npm", "install", "-g", f"--prefix={npm_prefix}", "@google/gemini-cli"], + ["npm", "install", "-g", f"--prefix={npm_prefix}", "@google/gemini-cli@nightly"], capture_output=True, text=True, env={**os.environ, "HOME": str(home)} ) @@ -91,7 +91,7 @@ # 3. Write settings.json with model preferences settings = { "theme": "Default", - "selectedAuthType": "api-key" + "selectedAuthType": "gemini-api-key" } settings_path = gemini_dir / "settings.json" From dadeea7dcec49b99328691300ee9628c9c25e0ba Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 14:45:37 -0500 Subject: [PATCH 009/382] fix: set Gemini model in settings.json to prevent default model error Gemini CLI defaults to gemini-2.5-flash-lite which doesn't exist on Databricks. Set model.name in settings.json to the configured model. Co-Authored-By: Claude Opus 4.6 --- setup_gemini.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup_gemini.py b/setup_gemini.py index fc76c517..9e6519b3 100644 --- a/setup_gemini.py +++ b/setup_gemini.py @@ -23,7 +23,7 @@ host = os.environ.get("DATABRICKS_HOST", "") token = os.environ.get("DATABRICKS_TOKEN", "") -gemini_model = os.environ.get("GEMINI_MODEL", "databricks-gemini-2-5-flash") +gemini_model = os.environ.get("GEMINI_MODEL", "databricks-gemini-3-1-pro") if not host or not token: print("Warning: DATABRICKS_HOST or DATABRICKS_TOKEN not set, skipping Gemini CLI config") @@ -88,10 +88,13 @@ env_path.chmod(0o600) print(f"Gemini CLI env configured: {env_path}") -# 3. Write settings.json with model preferences +# 3. Write settings.json with model preferences and auth settings = { "theme": "Default", - "selectedAuthType": "gemini-api-key" + "selectedAuthType": "gemini-api-key", + "model": { + "name": gemini_model + } } settings_path = gemini_dir / "settings.json" @@ -100,7 +103,5 @@ print("\nGemini CLI ready! Usage:") print(" gemini # Start Gemini CLI") -print(f" gemini -m gemini-2.5-flash # Use Gemini 2.5 Flash") -print(f" gemini -m gemini-2.5-pro # Use Gemini 2.5 Pro") print(f"\nEndpoint: {gemini_base_url}") print("Auth: Bearer token (Databricks PAT)") From babfc671aac40e8333f49493fead2393edec5f8b Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 15:05:46 -0500 Subject: [PATCH 010/382] feat: copy Claude skills into .gemini/skills directory Shares the Claude skills with Gemini CLI so both agents have access to the same Databricks and workflow skills. Co-Authored-By: Claude Opus 4.6 --- setup_gemini.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup_gemini.py b/setup_gemini.py index 9e6519b3..9c8e736d 100644 --- a/setup_gemini.py +++ b/setup_gemini.py @@ -12,6 +12,7 @@ """ import os import json +import shutil import subprocess from pathlib import Path @@ -101,6 +102,17 @@ settings_path.write_text(json.dumps(settings, indent=2)) print(f"Gemini CLI settings configured: {settings_path}") +# 4. Copy Claude skills into .gemini/skills for shared reference +claude_skills_dir = home / ".claude" / "skills" +gemini_skills_dir = gemini_dir / "skills" +if claude_skills_dir.exists(): + if gemini_skills_dir.exists(): + shutil.rmtree(gemini_skills_dir) + shutil.copytree(claude_skills_dir, gemini_skills_dir) + print(f"Skills copied: {claude_skills_dir} -> {gemini_skills_dir}") +else: + print(f"No Claude skills found at {claude_skills_dir}, skipping copy") + print("\nGemini CLI ready! Usage:") print(" gemini # Start Gemini CLI") print(f"\nEndpoint: {gemini_base_url}") From 52f60b5e27e5ae960593af264c1814c699a0ff13 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 15:28:43 -0500 Subject: [PATCH 011/382] chore: sync app.yaml.template with current app.yaml Add OpenCode and Gemini CLI setup scripts, remove hardcoded DATABRICKS_HOST, add gateway and model config placeholders. Co-Authored-By: Claude Opus 4.6 --- app.yaml.template | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/app.yaml.template b/app.yaml.template index 398491d7..edb349ea 100644 --- a/app.yaml.template +++ b/app.yaml.template @@ -1,13 +1,18 @@ command: - bash - -c - - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_databricks.py && python app.py" + - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && python app.py" env: - name: HOME value: /app/python/source_code - - name: DATABRICKS_HOST - value: https://.cloud.databricks.com - name: DATABRICKS_TOKEN valueFrom: DATABRICKS_TOKEN - name: ANTHROPIC_MODEL - value: databricks-claude-sonnet-4-5 + value: databricks-claude-opus-4-6 + - name: GEMINI_MODEL + value: databricks-gemini-3-1-pro + #OPTIONAL: Use the new Databricks AI Gateway if you have access (recommended), otherwise it will default to the older endpoint + - name: DATABRICKS_GATEWAY_HOST + value: https://.ai-gateway..cloud.databricks.com + - name: DATABRICKS_GATEWAY_TOKEN + valueFrom: DATABRICKS_GATEWAY_TOKEN From a7ff1306efd8e2201ed9755832a49a611c13f0fe Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 15:49:30 -0500 Subject: [PATCH 012/382] feat: add loading screen with snake game during setup Start Flask immediately and run setup scripts (micro, claude, opencode, gemini, databricks) in a background thread. Users see a terminal-themed loading page with a playable snake game and live progress panel instead of a dead page during the ~60-90s setup window. - Simplified app.yaml command to just `python app.py` - Added /api/setup-status endpoint for progress polling - Auto-transitions to terminal on setup complete (or after 4s on error) Co-Authored-By: Claude Opus 4.6 --- app.py | 102 +++++++++- app.yaml | 5 +- app.yaml.template | 5 +- static/loading.html | 457 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 556 insertions(+), 13 deletions(-) create mode 100644 static/loading.html diff --git a/app.py b/app.py index 0648546a..6def20eb 100644 --- a/app.py +++ b/app.py @@ -9,6 +9,7 @@ import threading import signal import time +import copy import logging from flask import Flask, send_from_directory, request, jsonify, session from collections import deque @@ -29,10 +30,79 @@ sessions = {} sessions_lock = threading.Lock() +# Setup state tracking +setup_lock = threading.Lock() +setup_state = { + "status": "pending", + "started_at": None, + "completed_at": None, + "error": None, + "steps": [ + {"id": "micro", "label": "Installing micro editor", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "claude", "label": "Configuring Claude CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "opencode", "label": "Configuring OpenCode CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "gemini", "label": "Configuring Gemini CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "databricks", "label": "Setting up Databricks CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + ] +} + + +def _update_step(step_id, **kwargs): + with setup_lock: + for step in setup_state["steps"]: + if step["id"] == step_id: + step.update(kwargs) + break + + +def _get_setup_state_snapshot(): + with setup_lock: + return copy.deepcopy(setup_state) + + # Single-user security: only the token owner can access the terminal app_owner = None +def _run_step(step_id, command): + _update_step(step_id, status="running", started_at=time.time()) + try: + env = os.environ.copy() + if not env.get("HOME") or env["HOME"] == "/": + env["HOME"] = "/app/python/source_code" + env.pop("DATABRICKS_CLIENT_ID", None) + env.pop("DATABRICKS_CLIENT_SECRET", None) + + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=300) + if result.returncode == 0: + _update_step(step_id, status="complete", completed_at=time.time()) + else: + err = result.stderr.strip() or result.stdout.strip() or "Unknown error" + _update_step(step_id, status="error", completed_at=time.time(), error=err[:500]) + except subprocess.TimeoutExpired: + _update_step(step_id, status="error", completed_at=time.time(), error="Timed out after 300s") + except Exception as e: + _update_step(step_id, status="error", completed_at=time.time(), error=str(e)) + + +def run_setup(): + with setup_lock: + setup_state["status"] = "running" + setup_state["started_at"] = time.time() + + _run_step("micro", ["bash", "-c", + "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) + _run_step("claude", ["python", "setup_claude.py"]) + _run_step("opencode", ["python", "setup_opencode.py"]) + _run_step("gemini", ["python", "setup_gemini.py"]) + _run_step("databricks", ["python", "setup_databricks.py"]) + + with setup_lock: + any_error = any(s["status"] == "error" for s in setup_state["steps"]) + setup_state["status"] = "error" if any_error else "complete" + setup_state["completed_at"] = time.time() + + def get_token_owner(): """Get the owner email from DATABRICKS_TOKEN at startup.""" try: @@ -162,8 +232,8 @@ def cleanup_stale_sessions(): @app.before_request def authorize_request(): """Check authorization before processing any request.""" - # Skip auth for health check - if request.path == "/health": + # Skip auth for health check and setup status + if request.path in ("/health", "/api/setup-status"): return None authorized, user = check_authorization() @@ -178,17 +248,30 @@ def authorize_request(): @app.route("/") def index(): + with setup_lock: + status = setup_state["status"] + if status in ("pending", "running"): + return send_from_directory("static", "loading.html") return send_from_directory("static", "index.html") +@app.route("/api/setup-status") +def get_setup_status(): + return jsonify(_get_setup_state_snapshot()) + + @app.route("/health") def health(): with sessions_lock: - return jsonify({ - "status": "healthy", - "active_sessions": len(sessions), - "session_timeout_seconds": SESSION_TIMEOUT_SECONDS - }) + session_count = len(sessions) + with setup_lock: + current_setup_status = setup_state["status"] + return jsonify({ + "status": "healthy", + "setup_status": current_setup_status, + "active_sessions": session_count, + "session_timeout_seconds": SESSION_TIMEOUT_SECONDS + }) @app.route("/api/session", methods=["POST"]) @@ -341,4 +424,9 @@ def close_session(): cleanup_thread.start() logger.info(f"Started session cleanup thread (timeout={SESSION_TIMEOUT_SECONDS}s, interval={CLEANUP_INTERVAL_SECONDS}s)") + # Start setup in background thread — Flask starts immediately + setup_thread = threading.Thread(target=run_setup, daemon=True, name="setup-thread") + setup_thread.start() + logger.info("Started background setup thread") + app.run(host="0.0.0.0", port=8000, threaded=True) diff --git a/app.yaml b/app.yaml index 48ed0d58..2a1cd681 100644 --- a/app.yaml +++ b/app.yaml @@ -1,7 +1,6 @@ command: - - bash - - -c - - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && python app.py" + - python + - app.py env: - name: HOME value: /app/python/source_code diff --git a/app.yaml.template b/app.yaml.template index edb349ea..612938a0 100644 --- a/app.yaml.template +++ b/app.yaml.template @@ -1,7 +1,6 @@ command: - - bash - - -c - - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && python app.py" + - python + - app.py env: - name: HOME value: /app/python/source_code diff --git a/static/loading.html b/static/loading.html new file mode 100644 index 00000000..02b105ba --- /dev/null +++ b/static/loading.html @@ -0,0 +1,457 @@ + + + + + +Claude Code on Databricks - Setting Up + + + + +
+

claude code on databricks

+
Setting up your environment...
+
+ +
+
+
// play while you wait
+
+ Score: 0 + High: 0 +
+ +
Arrow keys / WASD to move
+
+ +
+
// setup progress
+
+
Waiting to start...
+
+
+ + + + From 72b1a500f2871f304532bee85dd296e2d9911151 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 16:10:32 -0500 Subject: [PATCH 013/382] fix: auto-configure git identity from Databricks token owner Resolves the "please tell me who you are" prompt when committing from any CLI (Gemini, OpenCode, etc). Sets global git user.email and user.name from the DATABRICKS_TOKEN owner during setup. Co-Authored-By: Claude Opus 4.6 --- setup_claude.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/setup_claude.py b/setup_claude.py index c619452f..a4a2bc0c 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -90,7 +90,29 @@ projects_dir.mkdir(exist_ok=True) print(f"Projects directory: {projects_dir}") -# 5. Set up global git hooks directory (works for ALL repos including clones) +# 5. Set up global git identity from Databricks token owner +try: + from databricks.sdk import WorkspaceClient + db_host = os.environ.get("DATABRICKS_HOST") + db_token = os.environ.get("DATABRICKS_TOKEN") + if db_host and db_token: + w = WorkspaceClient(host=db_host, token=db_token, auth_type="pat") + me = w.current_user.me() + user_email = me.user_name + display_name = me.display_name or user_email.split("@")[0] + subprocess.run( + ["git", "config", "--global", "user.email", user_email], + capture_output=True + ) + subprocess.run( + ["git", "config", "--global", "user.name", display_name], + capture_output=True + ) + print(f"Git identity configured: {display_name} <{user_email}>") +except Exception as e: + print(f"Warning: Could not set git identity from token: {e}") + +# 6. Set up global git hooks directory (works for ALL repos including clones) global_hooks_dir = home / ".githooks" global_hooks_dir.mkdir(parents=True, exist_ok=True) From 2d788fd5b92cad72ec75d3ea8061b98a1c8a260c Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 16:19:13 -0500 Subject: [PATCH 014/382] fix: separate git config into its own setup step with direct file writes The git config subprocess calls inside setup_claude.py weren't persisting when run through _run_step. Now git identity and hooks are configured directly in app.py by writing ~/.gitconfig and ~/.githooks/post-commit as files (no subprocess git commands). This runs as the first setup step. Co-Authored-By: Claude Opus 4.6 --- app.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ setup_claude.py | 43 +++--------------------------------- 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/app.py b/app.py index 6def20eb..749692dd 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,7 @@ "completed_at": None, "error": None, "steps": [ + {"id": "git", "label": "Configuring git identity", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "micro", "label": "Installing micro editor", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "claude", "label": "Configuring Claude CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "opencode", "label": "Configuring OpenCode CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, @@ -85,11 +86,68 @@ def _run_step(step_id, command): _update_step(step_id, status="error", completed_at=time.time(), error=str(e)) +def _setup_git_config(): + """Configure git identity and hooks by writing files directly (no subprocess).""" + home = os.environ.get("HOME", "/app/python/source_code") + if not home or home == "/": + home = "/app/python/source_code" + + # Get user identity from Databricks token + user_email = None + display_name = None + try: + from databricks.sdk import WorkspaceClient + db_host = os.environ.get("DATABRICKS_HOST") + db_token = os.environ.get("DATABRICKS_TOKEN") + if db_host and db_token: + w = WorkspaceClient(host=db_host, token=db_token, auth_type="pat") + me = w.current_user.me() + user_email = me.user_name + display_name = me.display_name or user_email.split("@")[0] + except Exception as e: + logger.warning(f"Could not get user identity from token: {e}") + + # Write ~/.gitconfig directly (more reliable than subprocess git config) + gitconfig_path = os.path.join(home, ".gitconfig") + hooks_dir = os.path.join(home, ".githooks") + os.makedirs(hooks_dir, exist_ok=True) + + lines = [] + if user_email and display_name: + lines.append("[user]") + lines.append(f"\temail = {user_email}") + lines.append(f"\tname = {display_name}") + lines.append("[core]") + lines.append(f"\thooksPath = {hooks_dir}") + + with open(gitconfig_path, "w") as f: + f.write("\n".join(lines) + "\n") + logger.info(f"Git config written to {gitconfig_path}") + + # Write post-commit hook for workspace sync + post_commit = os.path.join(hooks_dir, "post-commit") + with open(post_commit, "w") as f: + f.write("#!/bin/bash\n") + f.write("# Auto-sync to Databricks Workspace on commit\n") + f.write("source /app/python/source_code/.venv/bin/activate\n") + f.write('python /app/python/source_code/sync_to_workspace.py "$(pwd)" &\n') + os.chmod(post_commit, 0o755) + logger.info(f"Post-commit hook written to {post_commit}") + + def run_setup(): with setup_lock: setup_state["status"] = "running" setup_state["started_at"] = time.time() + # Git config — done directly in Python, not as a subprocess + _update_step("git", status="running", started_at=time.time()) + try: + _setup_git_config() + _update_step("git", status="complete", completed_at=time.time()) + except Exception as e: + _update_step("git", status="error", completed_at=time.time(), error=str(e)) + _run_step("micro", ["bash", "-c", "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) _run_step("claude", ["python", "setup_claude.py"]) diff --git a/setup_claude.py b/setup_claude.py index a4a2bc0c..e7d3bd59 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -90,43 +90,6 @@ projects_dir.mkdir(exist_ok=True) print(f"Projects directory: {projects_dir}") -# 5. Set up global git identity from Databricks token owner -try: - from databricks.sdk import WorkspaceClient - db_host = os.environ.get("DATABRICKS_HOST") - db_token = os.environ.get("DATABRICKS_TOKEN") - if db_host and db_token: - w = WorkspaceClient(host=db_host, token=db_token, auth_type="pat") - me = w.current_user.me() - user_email = me.user_name - display_name = me.display_name or user_email.split("@")[0] - subprocess.run( - ["git", "config", "--global", "user.email", user_email], - capture_output=True - ) - subprocess.run( - ["git", "config", "--global", "user.name", display_name], - capture_output=True - ) - print(f"Git identity configured: {display_name} <{user_email}>") -except Exception as e: - print(f"Warning: Could not set git identity from token: {e}") - -# 6. Set up global git hooks directory (works for ALL repos including clones) -global_hooks_dir = home / ".githooks" -global_hooks_dir.mkdir(parents=True, exist_ok=True) - -post_commit_hook = global_hooks_dir / "post-commit" -post_commit_hook.write_text('''#!/bin/bash -# Auto-sync to Databricks Workspace on commit -source /app/python/source_code/.venv/bin/activate -python /app/python/source_code/sync_to_workspace.py "$(pwd)" & -''') -post_commit_hook.chmod(0o755) - -# Configure git to use global hooks for ALL repos (including clones) -subprocess.run( - ["git", "config", "--global", "core.hooksPath", str(global_hooks_dir)], - capture_output=True -) -print(f"Git hooks configured: {global_hooks_dir} (applies to all repos)") +# 5. Git identity and hooks are now configured by app.py's _setup_git_config() +# (runs directly in Python before setup_claude.py, writes ~/.gitconfig and ~/.githooks/) +print("Git identity and hooks: configured by app.py (skipping here)") From f913c5f80000602e73542eb1d09991e922c6daca Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 17:08:07 -0500 Subject: [PATCH 015/382] feat: replace Werkzeug dev server with gunicorn for production - Extract initialize_app() from __main__ block for gunicorn hook - Add gunicorn.conf.py (1 worker, 8 threads, gthread class) - Update app.yaml/template to use gunicorn app:app - Make post-commit hook robust: direct venv python, logging to ~/.sync.log Co-Authored-By: Claude Opus 4.6 --- app.py | 13 ++++++++++--- app.yaml | 2 +- app.yaml.template | 2 +- gunicorn.conf.py | 16 ++++++++++++++++ setup_claude.py | 16 +++++++++++++--- 5 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 gunicorn.conf.py diff --git a/app.py b/app.py index 0648546a..04882c93 100644 --- a/app.py +++ b/app.py @@ -323,8 +323,10 @@ def close_session(): return jsonify({"status": "ok"}) -if __name__ == "__main__": - +def initialize_app(): + """One-time init: detect owner, start cleanup thread.""" + global app_owner + # Remove OAuth credentials - force PAT auth only os.environ.pop("DATABRICKS_CLIENT_ID", None) os.environ.pop("DATABRICKS_CLIENT_SECRET", None) @@ -341,4 +343,9 @@ def close_session(): cleanup_thread.start() logger.info(f"Started session cleanup thread (timeout={SESSION_TIMEOUT_SECONDS}s, interval={CLEANUP_INTERVAL_SECONDS}s)") - app.run(host="0.0.0.0", port=8000, threaded=True) + +if __name__ == "__main__": + # Local dev only — production uses gunicorn + initialize_app() + port = int(os.environ.get("DATABRICKS_APP_PORT", 8000)) + app.run(host="0.0.0.0", port=port, threaded=True) diff --git a/app.yaml b/app.yaml index 48ed0d58..f0f8a44c 100644 --- a/app.yaml +++ b/app.yaml @@ -1,7 +1,7 @@ command: - bash - -c - - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && python app.py" + - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && gunicorn app:app" env: - name: HOME value: /app/python/source_code diff --git a/app.yaml.template b/app.yaml.template index edb349ea..e41ff1bf 100644 --- a/app.yaml.template +++ b/app.yaml.template @@ -1,7 +1,7 @@ command: - bash - -c - - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && python app.py" + - "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true && python setup_claude.py && python setup_opencode.py && python setup_gemini.py && python setup_databricks.py && gunicorn app:app" env: - name: HOME value: /app/python/source_code diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 00000000..bb80b378 --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,16 @@ +import os + +bind = f"0.0.0.0:{os.environ.get('DATABRICKS_APP_PORT', '8000')}" +workers = 1 # PTY fds + sessions dict are process-local +threads = 8 # Concurrent request handling (poll + input + resize) +worker_class = "gthread" +timeout = 30 +graceful_timeout = 10 # Databricks gives 15s after SIGTERM +accesslog = "-" +errorlog = "-" +loglevel = "info" + + +def post_worker_init(worker): + from app import initialize_app + initialize_app() diff --git a/setup_claude.py b/setup_claude.py index c619452f..ca3a6437 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -96,9 +96,19 @@ post_commit_hook = global_hooks_dir / "post-commit" post_commit_hook.write_text('''#!/bin/bash -# Auto-sync to Databricks Workspace on commit -source /app/python/source_code/.venv/bin/activate -python /app/python/source_code/sync_to_workspace.py "$(pwd)" & +# Auto-sync to Databricks Workspace on commit (works from any CLI: Claude, Gemini, OpenCode, etc.) +SYNC_LOG="$HOME/.sync.log" +echo "[post-commit] $(date +%H:%M:%S) hook triggered in $(pwd)" >> "$SYNC_LOG" + +# Use venv python directly (avoids fragile 'source activate') +VENV_PYTHON="/app/python/source_code/.venv/bin/python" +SYNC_SCRIPT="/app/python/source_code/sync_to_workspace.py" + +if [ -x "$VENV_PYTHON" ] && [ -f "$SYNC_SCRIPT" ]; then + "$VENV_PYTHON" "$SYNC_SCRIPT" "$(pwd)" >> "$SYNC_LOG" 2>&1 & +else + echo "[post-commit] $(date +%H:%M:%S) SKIP: venv=$VENV_PYTHON script=$SYNC_SCRIPT" >> "$SYNC_LOG" +fi ''') post_commit_hook.chmod(0o755) From db1ec7b01db65486e14b60cfb3bd028e43b7265a Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 17:33:51 -0500 Subject: [PATCH 016/382] fix: use nohup+disown in post-commit hook for Gemini CLI compatibility Gemini CLI kills the entire process group when git commit finishes, which terminates backgrounded (&) processes. nohup+disown detaches the sync process so it survives process group cleanup. Co-Authored-By: Claude Opus 4.6 --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index b1f55d4b..1046edc0 100644 --- a/app.py +++ b/app.py @@ -137,7 +137,7 @@ def _setup_git_config(): f.write('SYNC_SCRIPT="/app/python/source_code/sync_to_workspace.py"\n') f.write('\n') f.write('if [ -x "$VENV_PYTHON" ] && [ -f "$SYNC_SCRIPT" ]; then\n') - f.write(' "$VENV_PYTHON" "$SYNC_SCRIPT" "$(pwd)" >> "$SYNC_LOG" 2>&1 &\n') + f.write(' nohup "$VENV_PYTHON" "$SYNC_SCRIPT" "$(pwd)" >> "$SYNC_LOG" 2>&1 & disown\n') f.write('else\n') f.write(' echo "[post-commit] $(date +%H:%M:%S) SKIP: venv=$VENV_PYTHON script=$SYNC_SCRIPT" >> "$SYNC_LOG"\n') f.write('fi\n') From 3934ebb130868d39d10d83ed7b101525c3c242cd Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 18:19:25 -0500 Subject: [PATCH 017/382] docs: update README for Coding Agents on Databricks Apps rebrand Rewrote README to reflect multi-CLI support (Claude Code, Gemini CLI, OpenCode), loading screen, gunicorn production server, AI Gateway routing, and correct deployment workflow using databricks sync. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 3 + README.md | 317 ++++++++++++++++++++++++++---------------------------- 2 files changed, 154 insertions(+), 166 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a2241b12..a9ade757 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,3 +86,6 @@ Before starting any new project or documentation: - Databricks skills from [databricks-solutions/ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) - Development workflow skills from [obra/superpowers](https://github.com/obra/superpowers) + +# things to remember +Remember to never move .git folder to the workspace if you're running workspace import. \ No newline at end of file diff --git a/README.md b/README.md index 71d8934c..d0a261d9 100644 --- a/README.md +++ b/README.md @@ -1,205 +1,194 @@ -# claude-code-cli-bricks +# Coding Agents on Databricks Apps + ### What is it? -TL;DR: Claude Code on Databricks Apps for All Databricks Users 🚀 +TL;DR: Run Claude Code, Gemini CLI, and OpenCode on Databricks Apps - all from the browser. -A browser-based terminal emulator built with Flask and xterm.js, designed for cloud development environments with Databricks workspace integration and Claude Code CLI support. +A browser-based terminal emulator that gives every Databricks user access to AI coding agents, wired up to model serving endpoints on their workspace. No IDE setup, no local installs. ### Why now? On Jan 26. 2026, Andrej Karpathy made [this viral tweet](https://x.com/karpathy/status/2015883857489522876?s=46&t=tEsLJXJnGFIkaWs-Bhs1yA). Boris Cherny, the creator of claude code responded and said the following. ![alt text](image.png) -This app template opens this up for all Databricks Users! ❤️ +This app template opens this up for all Databricks Users! -No more pesky IDE setups, no bespoke tweaks. +No more pesky IDE setups, no bespoke tweaks. Just use it all on Databricks, from the browser. Wired up to model serving endpoints on your workspace. ## Features -✅ **Browser-based Terminal** - Full PTY support with xterm.js frontend - -✅ **Real-time I/O** - Responsive terminal with polling-based communication - -✅ **Graceful Session Cleanup** - Shell processes are properly terminated on exit, tab close, or timeout - -✅ **Terminal Resizing** - Dynamic resize support for responsive layouts - -✅ **Databricks Workspace Integration** - Auto-sync projects to Databricks Workspace on git commits - -✅ **Claude Code CLI** - Pre-configured to use Databricks hosted models as the API endpoint - -✅ **Configurable Model** - Switch between Claude models via `app.yaml` (default: `databricks-claude-opus-4-6`) +### 🤖 Coding Agents -✅ **Micro Editor** - Ships with [micro](https://micro-editor.github.io/), a modern terminal-based text editor +| Agent | Model | Description | +|-------|-------|-------------| +| 🟠 **Claude Code** | `databricks-claude-opus-4-6` | Anthropic's coding agent with 30 skills + 2 MCP servers | +| 🔵 **Gemini CLI** | `databricks-gemini-3-1-pro` | Google's coding agent with shared skills | +| 🟢 **OpenCode** | Configurable | Open-source coding agent with multi-provider support | -✅ **Databricks CLI** - Pre-configured with your PAT for immediate use +Every agent starts **preconfigured to your Databricks AI Gateway endpoint** — models, auth tokens, and base URLs are all wired up at boot. No API keys to manage, no manual config. -✅ **Single-User Security** - Only the token owner can access the terminal +### ⚡ Platform -✅ **MCP Servers** - DeepWiki for GitHub docs, Exa for web search +> 🎮 **Zero-config terminal in your browser.** Open the app, play snake while it sets up, start coding. -### 30 Pre-installed Skills +| | | +|---|---| +| 🖥️ **Browser Terminal** | Full PTY with xterm.js — resize, scroll, 256-color, the works | +| 🐍 **Loading Screen** | Snake game while 6 setup steps run in parallel | +| 🔄 **Workspace Sync** | Every `git commit` auto-syncs to `/Workspace/Users/{you}/projects/` | +| 👤 **Auto Git Identity** | `user.name` + `user.email` from your Databricks token | +| 🔒 **Single-User Security** | Only the PAT owner gets in. Everyone else sees 403. | +| 🌐 **AI Gateway** | Route all models through Databricks AI Gateway | +| ✏️ **Micro Editor** | [micro](https://micro-editor.github.io/) — a modern terminal editor | +| ⚙️ **Databricks CLI** | Pre-configured with your PAT, ready to go | +| 🚀 **Gunicorn** | Production-grade server with gthread workers | -✅ **Databricks Skills (16)** - Make building Databricks products simple. Create dashboards, jobs, pipelines, agents, and more with guided workflows that understand Databricks APIs and best practices. +--- -✅ **Superpowers Skills (14)** - Provide the agentic framework for Claude Code. Test-driven development, systematic debugging, brainstorming, parallel agent workflows, and structured planning for complex tasks. +### 🧠 30 Claude Code Skills -## Skill Details + + + + + +
-### Databricks Skills +**🔶 16 Databricks Skills** — [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) -From [databricks-solutions/ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit): +| | | +|---|---| +| 🤖 AI & Agents | agent-bricks, genie, mlflow-eval, model-serving | +| 📊 Analytics | aibi-dashboards, unity-catalog | +| 🔧 Data Eng | declarative-pipelines, jobs, synthetic-data | +| 💻 Dev | asset-bundles, app-apx, app-python, python-sdk, config | +| 📚 Reference | docs, pdf-generation | -| Category | Skills | -|----------|--------| -| AI & Agents | agent-bricks, databricks-genie, mlflow-evaluation, model-serving | -| Analytics | aibi-dashboards, databricks-unity-catalog | -| Data Engineering | spark-declarative-pipelines, databricks-jobs, synthetic-data-generation | -| Development | asset-bundles, databricks-app-apx, databricks-app-python, databricks-python-sdk, databricks-config | -| Reference | databricks-docs, unstructured-pdf-generation | + -### Development Workflow Skills +**⚡ 14 Superpowers Skills** — [obra/superpowers](https://github.com/obra/superpowers) -From [obra/superpowers](https://github.com/obra/superpowers): +| | | +|---|---| +| 🏗️ Build | brainstorming, writing-plans, executing-plans | +| 💻 Code | test-driven-dev, subagent-driven-dev | +| 🐛 Debug | systematic-debugging, verification | +| 👀 Review | requesting-review, receiving-review | +| 📦 Ship | finishing-branch, git-worktrees | +| 🔀 Meta | dispatching-agents, writing-skills, using-superpowers | -- brainstorming, test-driven-development, systematic-debugging, writing-plans -- verification-before-completion, executing-plans, dispatching-parallel-agents -- subagent-driven-development, using-git-worktrees, requesting-code-review -- receiving-code-review, finishing-a-development-branch, writing-skills, using-superpowers +
-## MCP Servers +--- -Pre-configured MCP servers for enhanced capabilities: +### 🔌 2 MCP Servers -| Server | Description | +| Server | What it does | |--------|-------------| -| **DeepWiki** | AI-powered documentation for any GitHub repository | -| **Exa** | Web search and code context retrieval | - -### Updating Skills - -Skills are bundled with the app. To update: - -1. Pull latest from [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) -2. Copy `databricks-skills/*` to `.claude/skills/` -3. For superpowers, pull latest from [obra/superpowers](https://github.com/obra/superpowers) and copy `skills/*` to `.claude/skills/` -4. Redeploy the app +| 📖 **DeepWiki** | Ask questions about any GitHub repo — gets AI-powered answers from the codebase | +| 🔍 **Exa** | Web search and code context retrieval for up-to-date information | ## Quick Start ### Prerequisites -- Python 3.11+ -- [uv](https://github.com/astral-sh/uv) (recommended) or pip +- A Databricks workspace with Model Serving endpoints enabled +- A Personal Access Token (PAT) +- Databricks CLI installed locally (for deployment) -## Deploying to Databricks +### Deploy to Databricks Apps -1. Clone this repo to your Databricks Workspace -2. Navigate to **Compute** → **Apps** -3. Click **Create App** and select **Custom App** -4. Point to the cloned repo and deploy +1. Clone this repo: + ```bash + git clone + cd coding-agents-on-databricks + ``` -### Installation -```bash -# Clone the repository -git clone https://github.com/your-username/claude-code-cli-bricks.git -cd claude-code-cli-bricks +2. Copy and configure `app.yaml`: + ```bash + cp app.yaml.template app.yaml + ``` + Edit `app.yaml` — set your `DATABRICKS_GATEWAY_HOST` or remove the gateway lines to fall back to direct model serving endpoints. -# Install dependencies -uv pip install -r requirements.txt -``` +3. Create the app and configure the `DATABRICKS_TOKEN` secret: + ```bash + databricks apps create + ``` + In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources), add your PAT as the `DATABRICKS_TOKEN` secret. If using AI Gateway, also add `DATABRICKS_GATEWAY_TOKEN`. + +4. Sync and deploy: + ```bash + databricks sync . /Workspace/Users//apps/ --watch=false + databricks apps deploy \ + --source-code-path /Workspace/Users//apps/ + ``` + +> **Important:** Use `databricks sync` (not `workspace import-dir`) to upload files. It respects `.gitignore` and handles the `.git` directory correctly. -### Running Locally +### Run Locally ```bash uv run python app.py ``` -Open http://localhost:8000 in your browser. - +Open http://localhost:8000. This starts Flask's dev server — production uses Gunicorn. ## Architecture ``` ┌─────────────────────┐ HTTP ┌─────────────────────┐ -│ Browser Client │◄────────────►│ Flask Backend │ +│ Browser Client │◄────────────►│ Gunicorn + Flask │ │ (xterm.js) │ Polling │ (PTY Manager) │ └─────────────────────┘ └─────────────────────┘ - │ - ▼ + │ │ + │ on first load │ on startup + ▼ ▼ +┌─────────────────────┐ ┌─────────────────────┐ +│ Loading Screen │ │ Background Setup │ +│ (snake game) │ │ (6 parallel steps)│ +└─────────────────────┘ └─────────────────────┘ + │ + ▼ ┌─────────────────────┐ │ Shell Process │ │ (/bin/bash) │ └─────────────────────┘ ``` +### Startup Flow + +1. Gunicorn starts, calls `initialize_app()` via `post_worker_init` hook +2. App immediately serves the loading screen (snake game) +3. Background thread runs setup steps: git config, micro editor, Claude CLI, OpenCode, Gemini CLI, Databricks CLI +4. `/api/setup-status` endpoint reports progress to the loading screen +5. Once complete, the loading screen transitions to the terminal UI + ### API Endpoints | Endpoint | Method | Description | |----------|--------|-------------| -| `/` | GET | Serves the terminal UI | -| `/health` | GET | Health check with session count | +| `/` | GET | Loading screen (during setup) or terminal UI | +| `/health` | GET | Health check with session count and setup status | +| `/api/setup-status` | GET | Setup progress for loading screen | | `/api/session` | POST | Create new terminal session | | `/api/input` | POST | Send input to terminal | | `/api/output` | POST | Poll for terminal output | | `/api/resize` | POST | Resize terminal dimensions | -| `/api/session/close` | POST | Gracefully close terminal session | - -## Project Structure - -``` -claude-code-cli-bricks/ -├── .claude/ -│ └── skills/ # 30 pre-installed skills -├── app.py # Flask backend with PTY management -├── app.yaml # Databricks Apps deployment config -├── app.yaml.template # Template for app.yaml configuration -├── CLAUDE.md # Claude Code welcome message -├── requirements.txt # Python dependencies -├── setup_claude.py # Claude Code CLI + MCP configuration -├── setup_databricks.py # Databricks CLI configuration -├── sync_to_workspace.py # Git hook for Databricks sync -├── static/ -│ ├── index.html # Terminal UI -│ └── lib/ # xterm.js library files -└── docs/ - └── plans/ # Design documentation -``` +| `/api/session/close` | POST | Close terminal session | ## Configuration -### Setting up app.yaml - -Copy the template and configure your Databricks workspace: - -```bash -cp app.yaml.template app.yaml -``` - -Edit `app.yaml` and replace `` with your Databricks workspace URL: - -```yaml -env: - - name: DATABRICKS_HOST - value: https://.cloud.databricks.com -``` - -The `DATABRICKS_HOST` is used by both: -- **Workspace sync** - To upload projects on git commits -- **Claude Code CLI** - As the Anthropic API endpoint (via Databricks serving endpoints) - -## Databricks Deployment - -This project is configured for deployment as a Databricks App. - ### Environment Variables -| Variable | Description | -|----------|-------------| -| `DATABRICKS_HOST` | Databricks workspace URL | -| `DATABRICKS_TOKEN` | Your Personal Access Token (PAT) | -| `ANTHROPIC_MODEL` | Model name (default: `databricks-claude-opus-4-6`) | +| Variable | Required | Description | +|----------|----------|-------------| +| `DATABRICKS_TOKEN` | Yes | Your Personal Access Token (secret) | +| `HOME` | Yes | Set to `/app/python/source_code` in app.yaml | +| `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | +| `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | +| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL (recommended). Falls back to direct model serving if unset | +| `DATABRICKS_GATEWAY_TOKEN` | No | AI Gateway token (secret, required if using gateway) | ### Security Model @@ -210,57 +199,53 @@ This is a **single-user app**. Each user deploys their own instance with their o 3. Only requests from the token owner are allowed 4. Other users see a 403 Forbidden error -This ensures your terminal session is private and uses your Databricks permissions. +### Gunicorn Configuration -### Create App +Production uses Gunicorn (`gunicorn.conf.py`) with: +- `workers=1` — PTY file descriptors and in-memory session state can't survive forking +- `threads=8` — Handles concurrent polling from the terminal client +- `worker_class=gthread` — Single process + thread pool +- `post_worker_init` hook calls `initialize_app()` to start setup -First, create the app in your Databricks workspace: +## Project Structure -```bash -databricks apps create xterm-terminal ``` - -### Deploy via CLI - -Deploy the code using the Databricks CLI: - -```bash -# 1. Import project files to workspace (wipe clean first for fresh deploy) -databricks workspace delete /Workspace/Users//xterm-experiment --recursive -databricks workspace import-dir . /Workspace/Users//xterm-experiment --overwrite - -# 2. Deploy the app -databricks apps deploy xterm-terminal --source-code-path /Workspace/Users//xterm-experiment +coding-agents-on-databricks/ +├── .claude/ +│ └── skills/ # 30 pre-installed Claude Code skills +├── app.py # Flask backend with PTY management + setup orchestration +├── app.yaml # Databricks Apps deployment config +├── app.yaml.template # Template for app.yaml +├── gunicorn.conf.py # Gunicorn production server config +├── CLAUDE.md # Claude Code instructions +├── requirements.txt # Python dependencies +├── setup_claude.py # Claude Code CLI + MCP configuration +├── setup_gemini.py # Gemini CLI configuration +├── setup_opencode.py # OpenCode CLI configuration +├── setup_databricks.py # Databricks CLI configuration +├── sync_to_workspace.py # Post-commit hook: sync to Databricks Workspace +├── install_micro.sh # Micro editor installer +├── static/ +│ ├── index.html # Terminal UI (xterm.js) +│ ├── loading.html # Loading screen with snake game +│ └── lib/ # xterm.js library files +└── docs/ + └── plans/ # Design documentation ``` -Replace `` with your Databricks username (e.g., `user@example.com`). - -Once the app is deployed, create a secret with your PAT in your Databricks Workspace. In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources), add the secret aliased as DATABRICKS_TOKEN. - -### Automatic Git Configuration - -When the app starts, it automatically configures git with your Databricks identity: -- **Email**: From your Databricks `userName` -- **Name**: From your Databricks `displayName` (or derived from email) - -This means commits made within the app will be attributed to your Databricks account. - ## Workspace Sync -When deployed, git commits automatically sync your projects to Databricks Workspace: +Git commits automatically sync projects to Databricks Workspace: ``` /Workspace/Users/{email}/projects/{project-name}/ ``` -This is enabled via a git post-commit hook configured by `setup_claude.py`. +The post-commit hook uses `nohup ... & disown` to ensure the sync process survives across all coding agents (Claude Code, Gemini CLI, OpenCode), since some agents kill the entire process group when a shell command finishes. ## Technologies -- **Backend**: Flask, Python PTY/termios +- **Backend**: Flask, Gunicorn (gthread), Python PTY/termios - **Frontend**: xterm.js, FitAddon -- **Integration**: Databricks SDK, Claude Agent SDK - -## License - -MIT +- **Agents**: Claude Code CLI, Gemini CLI, OpenCode +- **Integration**: Databricks SDK, Databricks AI Gateway From e6f64dfa26ce5a1dc867947342bc763f7bf44b74 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 18:24:20 -0500 Subject: [PATCH 018/382] refactor: unify gateway auth to use DATABRICKS_TOKEN everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove separate DATABRICKS_GATEWAY_TOKEN references — the AI Gateway now authenticates with the same DATABRICKS_TOKEN as direct model serving. Co-Authored-By: Claude Opus 4.6 --- README.md | 4 ++-- app.yaml.template | 4 ++-- setup_claude.py | 4 ++-- setup_gemini.py | 4 ++-- setup_opencode.py | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d0a261d9..bdca5ef7 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Every agent starts **preconfigured to your Databricks AI Gateway endpoint** — ```bash databricks apps create ``` - In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources), add your PAT as the `DATABRICKS_TOKEN` secret. If using AI Gateway, also add `DATABRICKS_GATEWAY_TOKEN`. + In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources), add your PAT as the `DATABRICKS_TOKEN` secret. If using AI Gateway, also add `DATABRICKS_TOKEN`. 4. Sync and deploy: ```bash @@ -188,7 +188,7 @@ Open http://localhost:8000. This starts Flask's dev server — production uses G | `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | | `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | | `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL (recommended). Falls back to direct model serving if unset | -| `DATABRICKS_GATEWAY_TOKEN` | No | AI Gateway token (secret, required if using gateway) | +| `DATABRICKS_TOKEN` | No | AI Gateway token (secret, required if using gateway) | ### Security Model diff --git a/app.yaml.template b/app.yaml.template index 1d9eb338..d6649f45 100644 --- a/app.yaml.template +++ b/app.yaml.template @@ -13,5 +13,5 @@ env: #OPTIONAL: Use the new Databricks AI Gateway if you have access (recommended), otherwise it will default to the older endpoint - name: DATABRICKS_GATEWAY_HOST value: https://.ai-gateway..cloud.databricks.com - - name: DATABRICKS_GATEWAY_TOKEN - valueFrom: DATABRICKS_GATEWAY_TOKEN + - name: DATABRICKS_TOKEN + valueFrom: DATABRICKS_TOKEN diff --git a/setup_claude.py b/setup_claude.py index e7d3bd59..db0b1fcc 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -19,9 +19,9 @@ databricks_host = os.environ.get("DATABRICKS_HOST", "").rstrip("/") if gateway_host: - gateway_token = os.environ.get("DATABRICKS_GATEWAY_TOKEN", "") + gateway_token = os.environ.get("DATABRICKS_TOKEN", "") if not gateway_token: - print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_GATEWAY_TOKEN missing, falling back to DATABRICKS_HOST") + print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") gateway_host = "" if gateway_host: diff --git a/setup_gemini.py b/setup_gemini.py index 9c8e736d..37edb267 100644 --- a/setup_gemini.py +++ b/setup_gemini.py @@ -35,9 +35,9 @@ # Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") -gateway_token = os.environ.get("DATABRICKS_GATEWAY_TOKEN", "") if gateway_host else "" +gateway_token = os.environ.get("DATABRICKS_TOKEN", "") if gateway_host else "" if gateway_host and not gateway_token: - print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_GATEWAY_TOKEN missing, falling back to DATABRICKS_HOST") + print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") gateway_host = "" if gateway_host: diff --git a/setup_opencode.py b/setup_opencode.py index 7b7c2469..260408cf 100644 --- a/setup_opencode.py +++ b/setup_opencode.py @@ -24,9 +24,9 @@ # Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to current gateway (DATABRICKS_HOST) gateway_host = os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/") -gateway_token = os.environ.get("DATABRICKS_GATEWAY_TOKEN", "") if gateway_host else "" +gateway_token = os.environ.get("DATABRICKS_TOKEN", "") if gateway_host else "" if gateway_host and not gateway_token: - print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_GATEWAY_TOKEN missing, falling back to DATABRICKS_HOST") + print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") gateway_host = "" if gateway_host: @@ -74,7 +74,7 @@ "name": "Databricks AI Gateway (MLflow)", "options": { "baseURL": f"{gateway_host}/mlflow/v1", - "apiKey": "{env:DATABRICKS_GATEWAY_TOKEN}" + "apiKey": "{env:DATABRICKS_TOKEN}" }, "models": { "databricks-claude-opus-4-6": { @@ -119,7 +119,7 @@ "name": "Databricks AI Gateway (OpenAI)", "options": { "baseURL": f"{gateway_host}/openai/v1", - "apiKey": "{env:DATABRICKS_GATEWAY_TOKEN}" + "apiKey": "{env:DATABRICKS_TOKEN}" }, "models": { "databricks-gpt-5-2-codex": { From 4204591c8434dad08ba11f5999ebedc3a2b2123e Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 18:25:55 -0500 Subject: [PATCH 019/382] chore: track app.yaml and remove DATABRICKS_GATEWAY_TOKEN from it Co-Authored-By: Claude Opus 4.6 --- .gitignore | 2 +- app.yaml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 7b753b0c..f18da9d2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ __pycache__/ venv/ # Workspace-specific config (use app.yaml.template) -app.yaml +# app.yaml # Git worktrees .worktrees/ diff --git a/app.yaml b/app.yaml index afa59234..17988fb1 100644 --- a/app.yaml +++ b/app.yaml @@ -13,5 +13,3 @@ env: #OPTIONAL: Move to the new Databricks Gateway if you have access (recommended), otherwise it will default to the older endpoint - name: DATABRICKS_GATEWAY_HOST value: https://6051921418418893.ai-gateway.staging.cloud.databricks.com - - name: DATABRICKS_GATEWAY_TOKEN - valueFrom: DATABRICKS_GATEWAY_TOKEN From 4c80f49946a6633e6ab1318d97a8613724d524cd Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 18:30:25 -0500 Subject: [PATCH 020/382] docs: update skill count to 39 and add refresh-skills feature 25 Databricks + 14 Superpowers skills. Added missing skills to the table (metric-views, structured-streaming, vector-search, lakebase, dbsql, zerobus-ingest, spark-python-data-source). Added skill refresh feature to platform table. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 2 +- README.md | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a9ade757..f39164f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # Claude Code on Databricks -Welcome! This environment comes pre-configured with 30 skills and 2 MCP servers. +Welcome! This environment comes pre-configured with 39 skills and 2 MCP servers. ## Skills (30 total) diff --git a/README.md b/README.md index bdca5ef7..bd6fe7bb 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Just use it all on Databricks, from the browser. Wired up to model serving endpo | Agent | Model | Description | |-------|-------|-------------| -| 🟠 **Claude Code** | `databricks-claude-opus-4-6` | Anthropic's coding agent with 30 skills + 2 MCP servers | +| 🟠 **Claude Code** | `databricks-claude-opus-4-6` | Anthropic's coding agent with 39 skills + 2 MCP servers | | 🔵 **Gemini CLI** | `databricks-gemini-3-1-pro` | Google's coding agent with shared skills | | 🟢 **OpenCode** | Configurable | Open-source coding agent with multi-provider support | @@ -43,24 +43,27 @@ Every agent starts **preconfigured to your Databricks AI Gateway endpoint** — | ✏️ **Micro Editor** | [micro](https://micro-editor.github.io/) — a modern terminal editor | | ⚙️ **Databricks CLI** | Pre-configured with your PAT, ready to go | | 🚀 **Gunicorn** | Production-grade server with gthread workers | +| 🔄 **Skill Refresh** | `/refresh-databricks-skills` pulls latest from [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) | --- -### 🧠 30 Claude Code Skills +### 🧠 39 Claude Code Skills
-**🔶 16 Databricks Skills** — [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) +**🔶 25 Databricks Skills** — [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) | | | |---|---| | 🤖 AI & Agents | agent-bricks, genie, mlflow-eval, model-serving | -| 📊 Analytics | aibi-dashboards, unity-catalog | -| 🔧 Data Eng | declarative-pipelines, jobs, synthetic-data | -| 💻 Dev | asset-bundles, app-apx, app-python, python-sdk, config | -| 📚 Reference | docs, pdf-generation | +| 📊 Analytics | aibi-dashboards, unity-catalog, metric-views | +| 🔧 Data Eng | declarative-pipelines, jobs, structured-streaming, synthetic-data, zerobus-ingest | +| 💻 Dev | asset-bundles, app-apx, app-python, python-sdk, config, spark-python-data-source | +| 🗄️ Storage | lakebase-autoscale, lakebase-provisioned, vector-search | +| 📚 Reference | docs, dbsql, pdf-generation | +| 🔄 Meta | refresh-databricks-skills | @@ -212,7 +215,7 @@ Production uses Gunicorn (`gunicorn.conf.py`) with: ``` coding-agents-on-databricks/ ├── .claude/ -│ └── skills/ # 30 pre-installed Claude Code skills +│ └── skills/ # 39 pre-installed Claude Code skills ├── app.py # Flask backend with PTY management + setup orchestration ├── app.yaml # Databricks Apps deployment config ├── app.yaml.template # Template for app.yaml From 984e02d88f065706da17369ef6545cd1b9547857 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 18:32:29 -0500 Subject: [PATCH 021/382] docs: stack skill tables vertically and rename to just Skills Co-Authored-By: Claude Opus 4.6 --- README.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index bd6fe7bb..ee973418 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Just use it all on Databricks, from the browser. Wired up to model serving endpo | Agent | Model | Description | |-------|-------|-------------| -| 🟠 **Claude Code** | `databricks-claude-opus-4-6` | Anthropic's coding agent with 39 skills + 2 MCP servers | +| 🟠 **Claude Code** | `databricks-claude-opus-4-6` | Anthropic's coding agent with 39 skills + 2 MCP servers (Claude Code) | | 🔵 **Gemini CLI** | `databricks-gemini-3-1-pro` | Google's coding agent with shared skills | | 🟢 **OpenCode** | Configurable | Open-source coding agent with multi-provider support | @@ -47,11 +47,7 @@ Every agent starts **preconfigured to your Databricks AI Gateway endpoint** — --- -### 🧠 39 Claude Code Skills - - - - - - -
+### 🧠 39 Skills **🔶 25 Databricks Skills** — [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) @@ -65,9 +61,6 @@ Every agent starts **preconfigured to your Databricks AI Gateway endpoint** — | 📚 Reference | docs, dbsql, pdf-generation | | 🔄 Meta | refresh-databricks-skills | - - **⚡ 14 Superpowers Skills** — [obra/superpowers](https://github.com/obra/superpowers) | | | @@ -79,10 +72,6 @@ Every agent starts **preconfigured to your Databricks AI Gateway endpoint** — | 📦 Ship | finishing-branch, git-worktrees | | 🔀 Meta | dispatching-agents, writing-skills, using-superpowers | -
- --- ### 🔌 2 MCP Servers @@ -215,7 +204,7 @@ Production uses Gunicorn (`gunicorn.conf.py`) with: ``` coding-agents-on-databricks/ ├── .claude/ -│ └── skills/ # 39 pre-installed Claude Code skills +│ └── skills/ # 39 pre-installed skills ├── app.py # Flask backend with PTY management + setup orchestration ├── app.yaml # Databricks Apps deployment config ├── app.yaml.template # Template for app.yaml From 4a17dd01d9cd24d23b461432959b07d24fcc114f Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 22 Feb 2026 18:38:33 -0500 Subject: [PATCH 022/382] fix: rename loading page title to Coding Agents on Databricks Co-Authored-By: Claude Opus 4.6 --- static/loading.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/loading.html b/static/loading.html index 02b105ba..523ebcef 100644 --- a/static/loading.html +++ b/static/loading.html @@ -3,7 +3,7 @@ -Claude Code on Databricks - Setting Up +Coding Agents on Databricks - Setting Up
Loading...
+
+ @@ -344,9 +359,24 @@ let currentThemeName = localStorage.getItem('terminal-theme-name') || null; let lastDarkTheme = localStorage.getItem('terminal-last-dark') || 'Dark'; let lastLightTheme = localStorage.getItem('terminal-last-light') || 'Light'; - let termInstance = null; - let fitAddonInstance = null; - let searchAddonInstance = null; + + // ── Pane Object Model ───────────────────────────────────────── + // Each pane: { id, element, term, fitAddon, searchAddon, sessionId, pollInterval } + let panes = []; + let activePaneId = null; + let paneIdCounter = 0; + + function getActivePane() { + return panes.find(p => p.id === activePaneId) || panes[0]; + } + + function focusPane(id) { + activePaneId = id; + panes.forEach(p => { + p.element.classList.toggle('active', p.id === id); + if (p.id === id) p.term.focus(); + }); + } // Resolve initial theme if (!currentThemeName || !themes[currentThemeName]) { @@ -362,13 +392,11 @@ document.body.style.background = preset.body; document.body.style.color = preset.type === 'dark' ? '#fff' : '#383a42'; document.getElementById('theme-toggle').textContent = preset.type === 'dark' ? '\u2600\uFE0F' : '\uD83C\uDF19'; - // Update overlay backgrounds for theme const overlayBg = preset.type === 'dark' ? 'rgba(30,30,30,0.9)' : 'rgba(245,245,245,0.9)'; document.getElementById('search-bar').style.background = overlayBg; document.getElementById('dictation-preview').style.background = overlayBg; - if (termInstance) { - termInstance.options.theme = preset.theme; - } + // Apply to all panes + panes.forEach(p => { p.term.options.theme = preset.theme; }); if (preset.type === 'dark') { lastDarkTheme = name; localStorage.setItem('terminal-last-dark', name); @@ -389,10 +417,8 @@ currentFontSize = Math.max(MIN_FONT_SIZE, Math.min(MAX_FONT_SIZE, size)); localStorage.setItem('terminal-font-size', currentFontSize); updateFontSizeDisplay(); - if (termInstance) { - termInstance.options.fontSize = currentFontSize; - refitAndResize(); - } + panes.forEach(p => { p.term.options.fontSize = currentFontSize; }); + refitAllPanes(); } // ── Font Family ──────────────────────────────────────────────── @@ -401,19 +427,17 @@ if (!family) return; currentFontFamily = name; localStorage.setItem('terminal-font-family', name); - if (termInstance) { - termInstance.options.fontFamily = family; - refitAndResize(); - } + panes.forEach(p => { p.term.options.fontFamily = family; }); + refitAllPanes(); document.getElementById('font-family-select').value = name; } - // ── Refit helper ─────────────────────────────────────────────── - function refitAndResize() { - if (fitAddonInstance && termInstance) { - fitAddonInstance.fit(); - sendResize(termInstance.cols, termInstance.rows); - } + // ── Refit all panes ───────────────────────────────────────────── + function refitAllPanes() { + panes.forEach(p => { + p.fitAddon.fit(); + if (p.sessionId) sendResize(p.term.cols, p.term.rows, p.sessionId); + }); } // ── Populate toolbar dropdowns ───────────────────────────────── @@ -453,7 +477,6 @@ toolbarTab.addEventListener('click', () => { const isOpen = toolbarWrapper.classList.toggle('open'); if (isOpen) { - // Wait for transition to get panel width, then offset tab requestAnimationFrame(() => { const w = toolbarPanel.offsetWidth; toolbarTab.style.right = w + 'px'; @@ -475,20 +498,22 @@ searchInput.focus(); searchInput.select(); } else { - if (searchAddonInstance) searchAddonInstance.clearDecorations(); - if (termInstance) termInstance.focus(); + const ap = getActivePane(); + if (ap && ap.searchAddon) ap.searchAddon.clearDecorations(); + if (ap) ap.term.focus(); } } function doSearch(direction) { - if (!searchAddonInstance) return; + const ap = getActivePane(); + if (!ap || !ap.searchAddon) return; const query = searchInput.value; if (!query) return; const opts = { decorations: { matchOverviewRuler: '#888', activeMatchColorOverviewRuler: '#ffb000' } }; if (direction === 'next') { - searchAddonInstance.findNext(query, opts); + ap.searchAddon.findNext(query, opts); } else { - searchAddonInstance.findPrevious(query, opts); + ap.searchAddon.findPrevious(query, opts); } } @@ -577,13 +602,15 @@ dictationInput.value = ''; dictationInterim.textContent = ''; dictationInterim.classList.remove('has-text'); - if (termInstance) termInstance.focus(); + const ap = getActivePane(); + if (ap) ap.term.focus(); } function sendDictation() { const text = dictationInput.value.trim(); - if (text && sessionId) { - sendInput(text); + const ap = getActivePane(); + if (text && ap && ap.sessionId) { + sendInput(text, ap.sessionId); } closeDictation(); } @@ -617,19 +644,33 @@ if (e.ctrlKey && e.shiftKey && e.key === 'F') { e.preventDefault(); toggleSearch(); return; } - // Alt+V (Option+V) : toggle voice dictation — use e.code because macOS Alt+V produces '√' + // Alt+V (Option+V) : toggle voice dictation if (e.altKey && !e.ctrlKey && !e.shiftKey && e.code === 'KeyV') { e.preventDefault(); if (dictationActive) closeDictation(); else startDictation(); return; } + // Ctrl+Shift+D : split pane + if (e.ctrlKey && e.shiftKey && e.key === 'D') { + e.preventDefault(); splitPane(); return; + } + // Ctrl+Shift+W : close active pane + if (e.ctrlKey && e.shiftKey && e.key === 'W') { + e.preventDefault(); closeActivePane(); return; + } + // Ctrl+Shift+] : next pane + if (e.ctrlKey && e.shiftKey && e.key === ']') { + e.preventDefault(); cyclePaneFocus('next'); return; + } + // Ctrl+Shift+[ : prev pane + if (e.ctrlKey && e.shiftKey && e.key === '[') { + e.preventDefault(); cyclePaneFocus('prev'); return; + } }); - // ── Session / IO ─────────────────────────────────────────────── + // ── Session / IO (parameterized by sessionId) ────────────────── const status = document.getElementById('status'); - let sessionId = null; - let pollInterval = null; async function createSession() { const resp = await fetch('/api/session', { method: 'POST' }); @@ -638,56 +679,203 @@ return data.session_id; } - async function sendInput(input) { - if (!sessionId) return; + async function sendInput(input, sid) { + if (!sid) return; await fetch('/api/input', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ session_id: sessionId, input: input }) + body: JSON.stringify({ session_id: sid, input: input }) }); } - async function sendResize(cols, rows) { - if (!sessionId) return; + async function sendResize(cols, rows, sid) { + if (!sid) return; await fetch('/api/resize', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ session_id: sessionId, cols: cols, rows: rows }) + body: JSON.stringify({ session_id: sid, cols: cols, rows: rows }) }); } - async function pollOutput(term) { - if (!sessionId) return; + async function pollOutput(pane) { + if (!pane.sessionId) return; try { const resp = await fetch('/api/output', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ session_id: sessionId }) + body: JSON.stringify({ session_id: pane.sessionId }) }); if (!resp.ok) { - cleanupSession(); - term.write('\r\n\x1b[31mSession ended.\x1b[0m\r\n'); + pane.term.write('\r\n\x1b[31mSession ended.\x1b[0m\r\n'); + cleanupPane(pane); return; } const data = await resp.json(); - if (data.output) term.write(data.output); + if (data.output) pane.term.write(data.output); if (data.exited) { - term.write('\r\n\x1b[33mShell process exited. You can close this tab.\x1b[0m\r\n'); - cleanupSession(); + pane.term.write('\r\n\x1b[33mShell process exited.\x1b[0m\r\n'); + cleanupPane(pane); } } catch (e) { console.error('Poll error:', e); } } - function cleanupSession() { - if (pollInterval) { clearInterval(pollInterval); pollInterval = null; } - if (sessionId) { - navigator.sendBeacon('/api/session/close', JSON.stringify({ session_id: sessionId })); - sessionId = null; + function cleanupPane(pane) { + if (pane.pollInterval) { clearInterval(pane.pollInterval); pane.pollInterval = null; } + if (pane.sessionId) { + navigator.sendBeacon('/api/session/close', JSON.stringify({ session_id: pane.sessionId })); + pane.sessionId = null; } } + function cleanupAllPanes() { + panes.forEach(p => cleanupPane(p)); + } + + // ── Pane Management ──────────────────────────────────────────── + async function createPane() { + const id = 'pane-' + (++paneIdCounter); + const container = document.getElementById('pane-container'); + const element = document.createElement('div'); + element.className = 'pane'; + element.id = id; + + // Add divider before second pane + if (panes.length === 1) { + const divider = document.createElement('div'); + divider.id = 'pane-divider'; + container.appendChild(divider); + setupDividerDrag(divider); + } + + container.appendChild(element); + + const term = new Terminal({ + cursorBlink: true, + fontSize: currentFontSize, + fontFamily: fontFamilies[currentFontFamily] || 'monospace', + theme: themes[currentThemeName].theme + }); + + const fitAddon = new FitAddon.FitAddon(); + term.loadAddon(fitAddon); + term.loadAddon(new WebLinksAddon.WebLinksAddon()); + + let searchAddon = null; + if (typeof SearchAddon !== 'undefined') { + searchAddon = new SearchAddon.SearchAddon(); + term.loadAddon(searchAddon); + } + + term.open(element); + fitAddon.fit(); + + const sid = await createSession(); + await sendResize(term.cols, term.rows, sid); + + term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); + term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n\r\n'); + + const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid, pollInterval: null }; + term.onData(data => sendInput(data, pane.sessionId)); + pane.pollInterval = setInterval(() => pollOutput(pane), 100); + + // Click to focus + element.addEventListener('mousedown', () => focusPane(id)); + + panes.push(pane); + focusPane(id); + + return pane; + } + + async function splitPane() { + if (panes.length >= 2) return; + status.textContent = 'Splitting...'; + status.style.display = ''; + try { + await createPane(); + // Reset flex for even split + panes.forEach(p => { p.element.style.flex = '1'; }); + refitAllPanes(); + status.style.display = 'none'; + } catch (e) { + status.textContent = 'Split failed: ' + e.message; + status.style.color = '#ff5555'; + } + } + + function closeActivePane() { + if (panes.length <= 1) return; + const ap = getActivePane(); + if (!ap) return; + + cleanupPane(ap); + ap.term.dispose(); + ap.element.remove(); + + // Remove divider + const divider = document.getElementById('pane-divider'); + if (divider) divider.remove(); + + panes = panes.filter(p => p.id !== ap.id); + + // Reset remaining pane to full width + if (panes.length === 1) { + panes[0].element.style.flex = '1'; + } + + focusPane(panes[0].id); + refitAllPanes(); + } + + function cyclePaneFocus(direction) { + if (panes.length <= 1) return; + const idx = panes.findIndex(p => p.id === activePaneId); + const next = direction === 'next' + ? (idx + 1) % panes.length + : (idx - 1 + panes.length) % panes.length; + focusPane(panes[next].id); + } + + // ── Divider Drag ─────────────────────────────────────────────── + function setupDividerDrag(divider) { + let dragging = false; + + divider.addEventListener('mousedown', e => { + e.preventDefault(); + dragging = true; + divider.classList.add('dragging'); + document.body.style.cursor = 'col-resize'; + document.body.style.userSelect = 'none'; + }); + + document.addEventListener('mousemove', e => { + if (!dragging || panes.length < 2) return; + const container = document.getElementById('pane-container'); + const rect = container.getBoundingClientRect(); + let pct = ((e.clientX - rect.left) / rect.width) * 100; + pct = Math.max(15, Math.min(85, pct)); + panes[0].element.style.flex = `0 0 ${pct}%`; + panes[1].element.style.flex = '1 1 0'; + refitAllPanes(); + }); + + document.addEventListener('mouseup', () => { + if (dragging) { + dragging = false; + divider.classList.remove('dragging'); + document.body.style.cursor = ''; + document.body.style.userSelect = ''; + refitAllPanes(); + } + }); + } + + // ── Split button ─────────────────────────────────────────────── + document.getElementById('split-btn').addEventListener('click', () => splitPane()); + // ── Init ─────────────────────────────────────────────────────── async function init() { try { @@ -696,45 +884,13 @@ if (typeof Terminal === 'undefined') throw new Error('xterm.js not loaded'); if (typeof FitAddon === 'undefined') throw new Error('FitAddon not loaded'); - const term = new Terminal({ - cursorBlink: true, - fontSize: currentFontSize, - fontFamily: fontFamilies[currentFontFamily] || 'monospace', - theme: themes[currentThemeName].theme - }); - termInstance = term; - - const fitAddon = new FitAddon.FitAddon(); - fitAddonInstance = fitAddon; - const webLinksAddon = new WebLinksAddon.WebLinksAddon(); - term.loadAddon(fitAddon); - term.loadAddon(webLinksAddon); - - // Load search addon - if (typeof SearchAddon !== 'undefined') { - const searchAddon = new SearchAddon.SearchAddon(); - searchAddonInstance = searchAddon; - term.loadAddon(searchAddon); - } - - term.open(document.getElementById('terminal')); - fitAddon.fit(); - - status.textContent = 'Creating session...'; - sessionId = await createSession(); - await sendResize(term.cols, term.rows); + await createPane(); status.textContent = 'Connected!'; setTimeout(() => { status.style.display = 'none'; }, 1000); - term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); - term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n\r\n'); - - term.onData(data => sendInput(data)); - pollInterval = setInterval(() => pollOutput(term), 100); - - window.addEventListener('resize', () => refitAndResize()); - window.addEventListener('beforeunload', () => cleanupSession()); + window.addEventListener('resize', () => refitAllPanes()); + window.addEventListener('beforeunload', () => cleanupAllPanes()); } catch (e) { status.textContent = 'Error: ' + e.message; From e0d07d83244f077a0b485215fccbc40f7c0e98f2 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 17:20:31 -0500 Subject: [PATCH 044/382] fix: strip CLAUDECODE env vars from PTY shell sessions Prevents "cannot be launched inside another Claude Code session" error when running claude in the browser terminal while the server was started from within a Claude Code session. Co-Authored-By: Claude Opus 4.6 --- app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app.py b/app.py index 2d7a0daa..dbf9bea4 100644 --- a/app.py +++ b/app.py @@ -374,6 +374,9 @@ def create_session(): # Set up environment for the shell shell_env = os.environ.copy() shell_env["TERM"] = "xterm-256color" + # Remove Claude Code env vars so the browser terminal isn't seen as nested + shell_env.pop("CLAUDECODE", None) + shell_env.pop("CLAUDE_CODE_SESSION", None) # Ensure HOME is set correctly if not shell_env.get("HOME") or shell_env["HOME"] == "/": shell_env["HOME"] = "/app/python/source_code" From bb502bd4890a5130b0b629d11fd7886c51dff7aa Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 17:24:44 -0500 Subject: [PATCH 045/382] feat: show split/close pane shortcuts in welcome message Co-Authored-By: Claude Opus 4.6 --- static/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index 443d43b5..f9aba939 100644 --- a/static/index.html +++ b/static/index.html @@ -775,7 +775,8 @@ await sendResize(term.cols, term.rows, sid); term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); - term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n\r\n'); + term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n'); + term.write('\x1b[90mCtrl+Shift+D split pane \u2502 Ctrl+Shift+W close pane\x1b[0m\r\n\r\n'); const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid, pollInterval: null }; term.onData(data => sendInput(data, pane.sessionId)); From 61333ee549be6fcc4fce835d08ef6d4dd72035c4 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 18:39:55 -0500 Subject: [PATCH 046/382] fix: pane cycle shortcuts use e.code for Ctrl+Shift+]/[ Shift+] produces '}' and Shift+[ produces '{', so e.key never matched. Use e.code (BracketRight/BracketLeft) instead. Co-Authored-By: Claude Opus 4.6 --- static/index.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/static/index.html b/static/index.html index f9aba939..7431b7fb 100644 --- a/static/index.html +++ b/static/index.html @@ -659,12 +659,12 @@ if (e.ctrlKey && e.shiftKey && e.key === 'W') { e.preventDefault(); closeActivePane(); return; } - // Ctrl+Shift+] : next pane - if (e.ctrlKey && e.shiftKey && e.key === ']') { + // Ctrl+Shift+] : next pane — use e.code because Shift+] produces '}' + if (e.ctrlKey && e.shiftKey && e.code === 'BracketRight') { e.preventDefault(); cyclePaneFocus('next'); return; } - // Ctrl+Shift+[ : prev pane - if (e.ctrlKey && e.shiftKey && e.key === '[') { + // Ctrl+Shift+[ : prev pane — use e.code because Shift+[ produces '{' + if (e.ctrlKey && e.shiftKey && e.code === 'BracketLeft') { e.preventDefault(); cyclePaneFocus('prev'); return; } }); From 6883c2c00c92c41f77d50378df2980d645a60f69 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 19:01:12 -0500 Subject: [PATCH 047/382] docs: rewrite README for new user appeal, add deployment guide Restructure README with hero tagline, badges, simplified agent descriptions, terminal features highlight, easy start deploy path (Git repo in Databricks UI), and collapsed details sections. Extract full deployment instructions to docs/deployment.md. Rename repo references to coding-agents-in-databricks. Co-Authored-By: Claude Opus 4.6 --- README.md | 233 ++++++++++++++++++++------------------------- app.yaml | 2 +- docs/deployment.md | 99 +++++++++++++++++++ 3 files changed, 205 insertions(+), 129 deletions(-) create mode 100644 docs/deployment.md diff --git a/README.md b/README.md index 1e59b65b..e7e4d7f3 100644 --- a/README.md +++ b/README.md @@ -1,133 +1,120 @@ # Coding Agents on Databricks Apps -### What is it? +[![Deploy to Databricks](https://img.shields.io/badge/Deploy-Databricks%20Apps-FF3621?logo=databricks&logoColor=white)](docs/deployment.md) +[![Agents](https://img.shields.io/badge/Agents-4%20included-green)]() +[![Skills](https://img.shields.io/badge/Skills-39%20built--in-blue)]() -TL;DR: Run Claude Code, Codex, Gemini CLI, and OpenCode on Databricks Apps - all from the browser. +> Run Claude Code, Codex, Gemini CLI, and OpenCode in your browser — zero setup, wired to your Databricks workspace. -A browser-based terminal emulator that gives every Databricks user access to AI coding agents, wired up to model serving endpoints on their workspace. No IDE setup, no local installs. + -### Why now? -On Jan 26. 2026, Andrej Karpathy made [this viral tweet](https://x.com/karpathy/status/2015883857489522876?s=46&t=tEsLJXJnGFIkaWs-Bhs1yA). Boris Cherny, the creator of claude code responded and said the following. -![alt text](image.png) - -This app template opens this up for all Databricks Users! - -No more pesky IDE setups, no bespoke tweaks. - -Just use it all on Databricks, from the browser. Wired up to model serving endpoints on your workspace. - -## Features +--- -### 🤖 Coding Agents +## What's Inside -| Agent | Model | Description | -|-------|-------|-------------| -| 🟠 **Claude Code** | `databricks-claude-opus-4-6` | Anthropic's coding agent with 39 skills + 2 MCP servers (Claude Code) | -| 🟣 **Codex** | `databricks-gpt-5-2` | OpenAI's coding agent with adapted instructions | -| 🔵 **Gemini CLI** | `databricks-gemini-3-1-pro` | Google's coding agent with shared skills | -| 🟢 **OpenCode** | Configurable | Open-source coding agent with multi-provider support | +🟠 **Claude Code** — Anthropic's coding agent with 39 Databricks skills + 2 MCP servers +🟣 **Codex** — OpenAI's coding agent, pre-configured for Databricks +🔵 **Gemini CLI** — Google's coding agent with shared skills +🟢 **OpenCode** — Open-source agent with multi-provider support -Every agent starts **preconfigured to your Databricks AI Gateway endpoint** — models, auth tokens, and base URLs are all wired up at boot. No API keys to manage, no manual config. +Every agent starts **pre-wired to your Databricks AI Gateway** — models, auth tokens, and base URLs are all configured at boot. No API keys to manage. -### ⚡ Platform +--- -> 🎮 **Zero-config terminal in your browser.** Open the app, play snake while it sets up, start coding. +## Terminal Features | | | |---|---| -| 🖥️ **Browser Terminal** | Full PTY with xterm.js — resize, scroll, 256-color, the works | -| 🐍 **Loading Screen** | Snake game while 6 setup steps run in parallel | +| 🎨 **8 Themes** | Dracula, Nord, Solarized, Monokai, GitHub Dark, and more | +| ✂️ **Split Panes** | Run two sessions side by side with a draggable divider | +| 🔍 **Search** | Find anything in your terminal history (Ctrl+Shift+F) | +| 🎤 **Voice Input** | Dictate commands with your mic (Option+V) | +| ⌨️ **Customizable** | Fonts, font sizes, themes — all persisted across sessions | +| 🐍 **Loading Screen** | Play snake while 6 setup steps run in parallel | | 🔄 **Workspace Sync** | Every `git commit` auto-syncs to `/Workspace/Users/{you}/projects/` | -| 👤 **Auto Git Identity** | `user.name` + `user.email` from your Databricks token | -| 🔒 **Single-User Security** | Only the PAT owner gets in. Everyone else sees 403. | -| 🌐 **AI Gateway** | Route all models through Databricks AI Gateway | -| ✏️ **Micro Editor** | [micro](https://micro-editor.github.io/) — a modern terminal editor | +| ✏️ **Micro Editor** | Modern terminal editor, pre-installed | | ⚙️ **Databricks CLI** | Pre-configured with your PAT, ready to go | -| 🚀 **Gunicorn** | Production-grade server with gthread workers | -| 🔄 **Skill Refresh** | `/refresh-databricks-skills` pulls latest from [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) | --- -### 🧠 39 Skills +## Quick Start -**🔶 25 Databricks Skills** — [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) +### Deploy to Databricks Apps -| | | -|---|---| -| 🤖 AI & Agents | agent-bricks, genie, mlflow-eval, model-serving | -| 📊 Analytics | aibi-dashboards, unity-catalog, metric-views | -| 🔧 Data Eng | declarative-pipelines, jobs, structured-streaming, synthetic-data, zerobus-ingest | -| 💻 Dev | asset-bundles, app-apx, app-python, python-sdk, config, spark-python-data-source | -| 🗄️ Storage | lakebase-autoscale, lakebase-provisioned, vector-search | -| 📚 Reference | docs, dbsql, pdf-generation | -| 🔄 Meta | refresh-databricks-skills | +1. Go to **Databricks → Apps → Create App** +2. Choose **Custom App** and connect this Git repo: + ``` + https://github.com/datasciencemonkey/coding-agents-in-databricks.git + ``` +3. Add your PAT as the `DATABRICKS_TOKEN` secret in **App Resources** +4. Deploy -**⚡ 14 Superpowers Skills** — [obra/superpowers](https://github.com/obra/superpowers) +That's it. Open the app URL and start coding. -| | | -|---|---| -| 🏗️ Build | brainstorming, writing-plans, executing-plans | -| 💻 Code | test-driven-dev, subagent-driven-dev | -| 🐛 Debug | systematic-debugging, verification | -| 👀 Review | requesting-review, receiving-review | -| 📦 Ship | finishing-branch, git-worktrees | -| 🔀 Meta | dispatching-agents, writing-skills, using-superpowers | +[→ Full deployment guide](docs/deployment.md) — environment variables, gateway config, and advanced options. + +### Run locally + +```bash +git clone https://github.com/datasciencemonkey/coding-agents-in-databricks.git +cd coding-agents-in-databricks +uv run python app.py +``` + +Open [http://localhost:8000](http://localhost:8000) — type `claude`, `codex`, `gemini`, or `opencode` to start coding. --- -### 🔌 2 MCP Servers +## Why This Exists -| Server | What it does | -|--------|-------------| -| 📖 **DeepWiki** | Ask questions about any GitHub repo — gets AI-powered answers from the codebase | -| 🔍 **Exa** | Web search and code context retrieval for up-to-date information | +On Jan 26, 2026, Andrej Karpathy made [this viral tweet](https://x.com/karpathy/status/2015883857489522876?s=46&t=tEsLJXJnGFIkaWs-Bhs1yA) about the future of coding. Boris Cherny, the creator of Claude Code, responded: -## Quick Start +![Boris Cherny's response](image.png) -### Prerequisites +This app template opens that vision up for every Databricks user — no IDE setup, no local installs. Just open the browser and start coding with AI. -- A Databricks workspace with Model Serving endpoints enabled -- A Personal Access Token (PAT) -- Databricks CLI installed locally (for deployment) +--- -### Deploy to Databricks Apps +
+🧠 All 39 Skills -1. Clone this repo: - ```bash - git clone - cd coding-agents-on-databricks - ``` +### Databricks Skills (25) — [ai-dev-kit](https://github.com/databricks-solutions/ai-dev-kit) -2. Copy and configure `app.yaml`: - ```bash - cp app.yaml.template app.yaml - ``` - Edit `app.yaml` — set your `DATABRICKS_GATEWAY_HOST` or remove the gateway lines to fall back to direct model serving endpoints. +| Category | Skills | +|----------|--------| +| AI & Agents | agent-bricks, genie, mlflow-eval, model-serving | +| Analytics | aibi-dashboards, unity-catalog, metric-views | +| Data Engineering | declarative-pipelines, jobs, structured-streaming, synthetic-data, zerobus-ingest | +| Development | asset-bundles, app-apx, app-python, python-sdk, config, spark-python-data-source | +| Storage | lakebase-autoscale, lakebase-provisioned, vector-search | +| Reference | docs, dbsql, pdf-generation | +| Meta | refresh-databricks-skills | -3. Create the app and configure the `DATABRICKS_TOKEN` secret: - ```bash - databricks apps create - ``` - In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources), add your PAT as the `DATABRICKS_TOKEN` secret. If using AI Gateway, also add `DATABRICKS_TOKEN`. +### Superpowers Skills (14) — [obra/superpowers](https://github.com/obra/superpowers) -4. Sync and deploy: - ```bash - databricks sync . /Workspace/Users//apps/ --watch=false - databricks apps deploy \ - --source-code-path /Workspace/Users//apps/ - ``` +| Category | Skills | +|----------|--------| +| Build | brainstorming, writing-plans, executing-plans | +| Code | test-driven-dev, subagent-driven-dev | +| Debug | systematic-debugging, verification | +| Review | requesting-review, receiving-review | +| Ship | finishing-branch, git-worktrees | +| Meta | dispatching-agents, writing-skills, using-superpowers | -> **Important:** Use `databricks sync` (not `workspace import-dir`) to upload files. It respects `.gitignore` and handles the `.git` directory correctly. +
-### Run Locally +
+🔌 2 MCP Servers -```bash -uv run python app.py -``` +| Server | What it does | +|--------|-------------| +| **DeepWiki** | Ask questions about any GitHub repo — gets AI-powered answers from the codebase | +| **Exa** | Web search and code context retrieval for up-to-date information | -Open http://localhost:8000. This starts Flask's dev server — production uses Gunicorn. +
-## Architecture +
+🏗️ Architecture ``` ┌─────────────────────┐ HTTP ┌─────────────────────┐ @@ -153,7 +140,7 @@ Open http://localhost:8000. This starts Flask's dev server — production uses G 1. Gunicorn starts, calls `initialize_app()` via `post_worker_init` hook 2. App immediately serves the loading screen (snake game) -3. Background thread runs setup steps: git config, micro editor, Claude CLI, Codex CLI, OpenCode, Gemini CLI, Databricks CLI +3. Background thread runs setup: git config, micro editor, Claude CLI, Codex CLI, OpenCode, Gemini CLI, Databricks CLI 4. `/api/setup-status` endpoint reports progress to the loading screen 5. Once complete, the loading screen transitions to the terminal UI @@ -170,7 +157,10 @@ Open http://localhost:8000. This starts Flask's dev server — production uses G | `/api/resize` | POST | Resize terminal dimensions | | `/api/session/close` | POST | Close terminal session | -## Configuration +
+ +
+⚙️ Configuration ### Environment Variables @@ -181,66 +171,53 @@ Open http://localhost:8000. This starts Flask's dev server — production uses G | `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | | `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-2`) | | `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | -| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL (recommended). Falls back to direct model serving if unset | -| `DATABRICKS_TOKEN` | No | AI Gateway token (secret, required if using gateway) | +| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL (recommended) | ### Security Model -This is a **single-user app**. Each user deploys their own instance with their own PAT: +Single-user app — each user deploys their own instance with their own PAT. Only the token owner can access the terminal. Everyone else sees 403. -1. The `DATABRICKS_TOKEN` in `app.yaml` identifies the owner -2. At startup, the app determines the token owner via Databricks API -3. Only requests from the token owner are allowed -4. Other users see a 403 Forbidden error +### Gunicorn -### Gunicorn Configuration +Production uses `workers=1` (PTY state is process-local), `threads=8` (concurrent polling), `gthread` worker class. -Production uses Gunicorn (`gunicorn.conf.py`) with: -- `workers=1` — PTY file descriptors and in-memory session state can't survive forking -- `threads=8` — Handles concurrent polling from the terminal client -- `worker_class=gthread` — Single process + thread pool -- `post_worker_init` hook calls `initialize_app()` to start setup +
-## Project Structure +
+📁 Project Structure ``` -coding-agents-on-databricks/ -├── .claude/ -│ └── skills/ # 39 pre-installed skills -├── app.py # Flask backend with PTY management + setup orchestration -├── app.yaml # Databricks Apps deployment config -├── app.yaml.template # Template for app.yaml +coding-agents-in-databricks/ +├── app.py # Flask backend + PTY management + setup orchestration +├── app.yaml.template # Databricks Apps deployment config template ├── gunicorn.conf.py # Gunicorn production server config -├── CLAUDE.md # Claude Code instructions ├── requirements.txt # Python dependencies ├── setup_claude.py # Claude Code CLI + MCP configuration ├── setup_codex.py # Codex CLI configuration ├── setup_gemini.py # Gemini CLI configuration -├── setup_opencode.py # OpenCode CLI configuration +├── setup_opencode.py # OpenCode configuration ├── setup_databricks.py # Databricks CLI configuration -├── sync_to_workspace.py # Post-commit hook: sync to Databricks Workspace +├── sync_to_workspace.py # Post-commit hook: sync to Workspace ├── install_micro.sh # Micro editor installer ├── static/ -│ ├── index.html # Terminal UI (xterm.js) +│ ├── index.html # Terminal UI (xterm.js + split panes) │ ├── loading.html # Loading screen with snake game │ └── lib/ # xterm.js library files +├── .claude/ +│ └── skills/ # 39 pre-installed skills └── docs/ + ├── deployment.md # Full Databricks Apps deployment guide └── plans/ # Design documentation ``` -## Workspace Sync +
-Git commits automatically sync projects to Databricks Workspace: +--- -``` -/Workspace/Users/{email}/projects/{project-name}/ -``` +## Technologies -The post-commit hook uses `nohup ... & disown` to ensure the sync process survives across all coding agents (Claude Code, Codex, Gemini CLI, OpenCode), since some agents kill the entire process group when a shell command finishes. +Flask · Gunicorn · xterm.js · Python PTY · Databricks SDK · Databricks AI Gateway -## Technologies +--- -- **Backend**: Flask, Gunicorn (gthread), Python PTY/termios -- **Frontend**: xterm.js, FitAddon -- **Agents**: Claude Code CLI, Codex CLI, Gemini CLI, OpenCode -- **Integration**: Databricks SDK, Databricks AI Gateway +*Built with Claude Code on Databricks.* diff --git a/app.yaml b/app.yaml index 7e634a43..4c0dc5c1 100644 --- a/app.yaml +++ b/app.yaml @@ -14,6 +14,6 @@ env: value: databricks-gpt-5-2 #OPTIONAL: Move to the new Databricks Gateway if you have access (recommended), otherwise it will default to the older endpoint - name: DATABRICKS_GATEWAY_HOST - value: https://6051921418418893.ai-gateway.staging.cloud.databricks.com + valueFrom: DATABRICKS_GATEWAY_HOST - name: CLAUDE_CODE_DISABLE_AUTO_MEMORY value: 0 diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 00000000..092b4a4b --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,99 @@ +# Deploy to Databricks Apps + +## Prerequisites + +- A Databricks workspace with Model Serving endpoints enabled +- A Personal Access Token (PAT) + +## Easy Start (Git Repo) + +The simplest way — no CLI, no cloning, everything stays in the Databricks UI. + +1. Go to **Databricks → Apps → Create App** +2. Choose **Custom App** and connect this Git repo: + ``` + https://github.com/datasciencemonkey/coding-agents-in-databricks.git + ``` +3. In the **App Resources** tab, add your PAT as the `DATABRICKS_TOKEN` secret +4. Click **Deploy** + +The app pulls the code directly from Git. To update later, just re-deploy — it picks up the latest from the repo. + +> **Optional:** If you use [Databricks AI Gateway](https://docs.databricks.com/aws/en/ai-gateway/), also add `DATABRICKS_GATEWAY_HOST` as a secret or environment variable. Otherwise the app falls back to direct model serving endpoints. + +## Alternative: Deploy with CLI + +If you prefer working from the terminal or need more control: + +### 1. Clone the repo into your workspace + +```bash +databricks repos create \ + --url https://github.com/datasciencemonkey/coding-agents-in-databricks.git \ + --path /Workspace/Users//apps/coding-agents-in-databricks +``` + +### 2. Configure `app.yaml` + +In the cloned workspace folder, copy the template and edit it: + +```bash +cp app.yaml.template app.yaml +``` + +Set your `DATABRICKS_GATEWAY_HOST`, or remove the gateway lines to fall back to direct model serving endpoints. + +### 3. Create the app and add your token + +```bash +databricks apps create +``` + +In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databricks-apps/resources), add your PAT as the `DATABRICKS_TOKEN` secret. + +### 4. Deploy + +```bash +databricks apps deploy \ + --source-code-path /Workspace/Users//apps/coding-agents-in-databricks +``` + +> **Tip:** To update later, just `git pull` in the workspace repo and re-deploy. + +## Environment Variables + +| Variable | Required | Description | +|----------|----------|-------------| +| `DATABRICKS_TOKEN` | Yes | Your Personal Access Token (secret) | +| `HOME` | Yes | Set to `/app/python/source_code` in app.yaml | +| `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | +| `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-2`) | +| `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | +| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL (recommended). Falls back to direct model serving if unset | + +## Security Model + +This is a **single-user app**. Each user deploys their own instance with their own PAT: + +1. The `DATABRICKS_TOKEN` in `app.yaml` identifies the owner +2. At startup, the app determines the token owner via Databricks API +3. Only requests from the token owner are allowed +4. Other users see a 403 Forbidden error + +## Gunicorn Configuration + +Production uses Gunicorn (`gunicorn.conf.py`) with: +- `workers=1` — PTY file descriptors and in-memory session state can't survive forking +- `threads=8` — Handles concurrent polling from the terminal client +- `worker_class=gthread` — Single process + thread pool +- `post_worker_init` hook calls `initialize_app()` to start setup + +## Workspace Sync + +Git commits automatically sync projects to Databricks Workspace: + +``` +/Workspace/Users/{email}/projects/{project-name}/ +``` + +The post-commit hook uses `nohup ... & disown` to ensure the sync process survives across all coding agents, since some agents kill the entire process group when a shell command finishes. From 1a4bffe053b106bac0b98018b0e0d2de7e1fa733 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 19:01:36 -0500 Subject: [PATCH 048/382] docs: update repo name to coding-agents-databricks-apps Co-Authored-By: Claude Opus 4.6 --- README.md | 8 ++++---- docs/deployment.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e7e4d7f3..6087a16c 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Every agent starts **pre-wired to your Databricks AI Gateway** — models, auth 1. Go to **Databricks → Apps → Create App** 2. Choose **Custom App** and connect this Git repo: ``` - https://github.com/datasciencemonkey/coding-agents-in-databricks.git + https://github.com/datasciencemonkey/coding-agents-databricks-apps.git ``` 3. Add your PAT as the `DATABRICKS_TOKEN` secret in **App Resources** 4. Deploy @@ -56,8 +56,8 @@ That's it. Open the app URL and start coding. ### Run locally ```bash -git clone https://github.com/datasciencemonkey/coding-agents-in-databricks.git -cd coding-agents-in-databricks +git clone https://github.com/datasciencemonkey/coding-agents-databricks-apps.git +cd coding-agents-databricks-apps uv run python app.py ``` @@ -187,7 +187,7 @@ Production uses `workers=1` (PTY state is process-local), `threads=8` (concurren 📁 Project Structure ``` -coding-agents-in-databricks/ +coding-agents-databricks-apps/ ├── app.py # Flask backend + PTY management + setup orchestration ├── app.yaml.template # Databricks Apps deployment config template ├── gunicorn.conf.py # Gunicorn production server config diff --git a/docs/deployment.md b/docs/deployment.md index 092b4a4b..85659aad 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -12,7 +12,7 @@ The simplest way — no CLI, no cloning, everything stays in the Databricks UI. 1. Go to **Databricks → Apps → Create App** 2. Choose **Custom App** and connect this Git repo: ``` - https://github.com/datasciencemonkey/coding-agents-in-databricks.git + https://github.com/datasciencemonkey/coding-agents-databricks-apps.git ``` 3. In the **App Resources** tab, add your PAT as the `DATABRICKS_TOKEN` secret 4. Click **Deploy** @@ -29,8 +29,8 @@ If you prefer working from the terminal or need more control: ```bash databricks repos create \ - --url https://github.com/datasciencemonkey/coding-agents-in-databricks.git \ - --path /Workspace/Users//apps/coding-agents-in-databricks + --url https://github.com/datasciencemonkey/coding-agents-databricks-apps.git \ + --path /Workspace/Users//apps/coding-agents-databricks-apps ``` ### 2. Configure `app.yaml` @@ -55,7 +55,7 @@ In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databric ```bash databricks apps deploy \ - --source-code-path /Workspace/Users//apps/coding-agents-in-databricks + --source-code-path /Workspace/Users//apps/coding-agents-databricks-apps ``` > **Tip:** To update later, just `git pull` in the workspace repo and re-deploy. From a7778d9e500e1cd4e4acf085077c5e981b9d1bc3 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 19:01:52 -0500 Subject: [PATCH 049/382] docs: revert repo name to coding-agents-in-databricks Co-Authored-By: Claude Opus 4.6 --- README.md | 8 ++++---- docs/deployment.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6087a16c..e7e4d7f3 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Every agent starts **pre-wired to your Databricks AI Gateway** — models, auth 1. Go to **Databricks → Apps → Create App** 2. Choose **Custom App** and connect this Git repo: ``` - https://github.com/datasciencemonkey/coding-agents-databricks-apps.git + https://github.com/datasciencemonkey/coding-agents-in-databricks.git ``` 3. Add your PAT as the `DATABRICKS_TOKEN` secret in **App Resources** 4. Deploy @@ -56,8 +56,8 @@ That's it. Open the app URL and start coding. ### Run locally ```bash -git clone https://github.com/datasciencemonkey/coding-agents-databricks-apps.git -cd coding-agents-databricks-apps +git clone https://github.com/datasciencemonkey/coding-agents-in-databricks.git +cd coding-agents-in-databricks uv run python app.py ``` @@ -187,7 +187,7 @@ Production uses `workers=1` (PTY state is process-local), `threads=8` (concurren 📁 Project Structure ``` -coding-agents-databricks-apps/ +coding-agents-in-databricks/ ├── app.py # Flask backend + PTY management + setup orchestration ├── app.yaml.template # Databricks Apps deployment config template ├── gunicorn.conf.py # Gunicorn production server config diff --git a/docs/deployment.md b/docs/deployment.md index 85659aad..092b4a4b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -12,7 +12,7 @@ The simplest way — no CLI, no cloning, everything stays in the Databricks UI. 1. Go to **Databricks → Apps → Create App** 2. Choose **Custom App** and connect this Git repo: ``` - https://github.com/datasciencemonkey/coding-agents-databricks-apps.git + https://github.com/datasciencemonkey/coding-agents-in-databricks.git ``` 3. In the **App Resources** tab, add your PAT as the `DATABRICKS_TOKEN` secret 4. Click **Deploy** @@ -29,8 +29,8 @@ If you prefer working from the terminal or need more control: ```bash databricks repos create \ - --url https://github.com/datasciencemonkey/coding-agents-databricks-apps.git \ - --path /Workspace/Users//apps/coding-agents-databricks-apps + --url https://github.com/datasciencemonkey/coding-agents-in-databricks.git \ + --path /Workspace/Users//apps/coding-agents-in-databricks ``` ### 2. Configure `app.yaml` @@ -55,7 +55,7 @@ In the [App Resources tab](https://docs.databricks.com/aws/en/dev-tools/databric ```bash databricks apps deploy \ - --source-code-path /Workspace/Users//apps/coding-agents-databricks-apps + --source-code-path /Workspace/Users//apps/coding-agents-in-databricks ``` > **Tip:** To update later, just `git pull` in the workspace repo and re-deploy. From 40c09ab40dd885d0a13d468f96336f845d1e728f Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 19:05:18 -0500 Subject: [PATCH 050/382] docs: fix agent line breaks, remove footer tagline Co-Authored-By: Claude Opus 4.6 --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e7e4d7f3..debf9338 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,11 @@ ## What's Inside 🟠 **Claude Code** — Anthropic's coding agent with 39 Databricks skills + 2 MCP servers + 🟣 **Codex** — OpenAI's coding agent, pre-configured for Databricks + 🔵 **Gemini CLI** — Google's coding agent with shared skills + 🟢 **OpenCode** — Open-source agent with multi-provider support Every agent starts **pre-wired to your Databricks AI Gateway** — models, auth tokens, and base URLs are all configured at boot. No API keys to manage. @@ -216,8 +219,4 @@ coding-agents-in-databricks/ ## Technologies -Flask · Gunicorn · xterm.js · Python PTY · Databricks SDK · Databricks AI Gateway - ---- - -*Built with Claude Code on Databricks.* +Flask · Gunicorn · xterm.js · Python PTY · Databricks SDK · Databricks AI Gateway \ No newline at end of file From becd94cacc208bf94099671e3e56c16a30fc84d8 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Feb 2026 19:06:48 -0500 Subject: [PATCH 051/382] docs: link Agents and Skills badges to README sections Co-Authored-By: Claude Opus 4.6 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index debf9338..1d52c7ed 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Coding Agents on Databricks Apps [![Deploy to Databricks](https://img.shields.io/badge/Deploy-Databricks%20Apps-FF3621?logo=databricks&logoColor=white)](docs/deployment.md) -[![Agents](https://img.shields.io/badge/Agents-4%20included-green)]() -[![Skills](https://img.shields.io/badge/Skills-39%20built--in-blue)]() +[![Agents](https://img.shields.io/badge/Agents-4%20included-green)](#whats-inside) +[![Skills](https://img.shields.io/badge/Skills-39%20built--in-blue)](#-all-39-skills) > Run Claude Code, Codex, Gemini CLI, and OpenCode in your browser — zero setup, wired to your Databricks workspace. From f62abcda699ff070df178659ddca792dd5f1e493 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 1 Mar 2026 11:42:44 -0500 Subject: [PATCH 052/382] feat: add pane toolbar buttons and switch pane shortcut hint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add close pane (✕) and switch pane (⇆) buttons to the toolbar that appear dynamically when split panes are active. Update the welcome message to include Ctrl+Shift+] switch pane shortcut. Closes #26 Co-Authored-By: Claude Opus 4.6 --- static/index.html | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/static/index.html b/static/index.html index 7431b7fb..eaad4354 100644 --- a/static/index.html +++ b/static/index.html @@ -221,6 +221,8 @@ 🎤 + + @@ -776,7 +778,7 @@ term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n'); - term.write('\x1b[90mCtrl+Shift+D split pane \u2502 Ctrl+Shift+W close pane\x1b[0m\r\n\r\n'); + term.write('\x1b[90mCtrl+Shift+D split pane \u2502 Ctrl+Shift+W close pane \u2502 Ctrl+Shift+] switch pane\x1b[0m\r\n\r\n'); const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid, pollInterval: null }; term.onData(data => sendInput(data, pane.sessionId)); @@ -800,6 +802,7 @@ // Reset flex for even split panes.forEach(p => { p.element.style.flex = '1'; }); refitAllPanes(); + updatePaneButtons(); status.style.display = 'none'; } catch (e) { status.textContent = 'Split failed: ' + e.message; @@ -829,6 +832,7 @@ focusPane(panes[0].id); refitAllPanes(); + updatePaneButtons(); } function cyclePaneFocus(direction) { @@ -874,8 +878,17 @@ }); } - // ── Split button ─────────────────────────────────────────────── + // ── Pane toolbar buttons ──────────────────────────────────────── document.getElementById('split-btn').addEventListener('click', () => splitPane()); + document.getElementById('close-pane-btn').addEventListener('click', () => closeActivePane()); + document.getElementById('next-pane-btn').addEventListener('click', () => cyclePaneFocus('next')); + + function updatePaneButtons() { + const multi = panes.length > 1; + document.getElementById('close-pane-btn').style.display = multi ? '' : 'none'; + document.getElementById('next-pane-btn').style.display = multi ? '' : 'none'; + document.getElementById('split-btn').style.display = panes.length >= 2 ? 'none' : ''; + } // ── Init ─────────────────────────────────────────────────────── async function init() { From e6f20c6b629975c35c16e508f7f9be210f855dd2 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 1 Mar 2026 11:43:25 -0500 Subject: [PATCH 053/382] docs: update README for GitHub template repo Add "Use this template" badge linking to repo generation. Update Quick Start to lead with the template workflow. Update "Why This Exists" copy to reference the template. Closes #27 Co-Authored-By: Claude Opus 4.6 --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1d52c7ed..dbd76b41 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Coding Agents on Databricks Apps +[![Use this template](https://img.shields.io/badge/Use%20this%20template-2ea44f?logo=github)](https://github.com/datasciencemonkey/coding-agents-databricks-apps/generate) [![Deploy to Databricks](https://img.shields.io/badge/Deploy-Databricks%20Apps-FF3621?logo=databricks&logoColor=white)](docs/deployment.md) [![Agents](https://img.shields.io/badge/Agents-4%20included-green)](#whats-inside) [![Skills](https://img.shields.io/badge/Skills-39%20built--in-blue)](#-all-39-skills) @@ -44,13 +45,11 @@ Every agent starts **pre-wired to your Databricks AI Gateway** — models, auth ### Deploy to Databricks Apps -1. Go to **Databricks → Apps → Create App** -2. Choose **Custom App** and connect this Git repo: - ``` - https://github.com/datasciencemonkey/coding-agents-in-databricks.git - ``` -3. Add your PAT as the `DATABRICKS_TOKEN` secret in **App Resources** -4. Deploy +1. Click [**Use this template**](https://github.com/datasciencemonkey/coding-agents-databricks-apps/generate) to create your own repo +2. Go to **Databricks → Apps → Create App** +3. Choose **Custom App** and connect your new repo +4. Add your PAT as the `DATABRICKS_TOKEN` secret in **App Resources** +5. Deploy That's it. Open the app URL and start coding. @@ -74,7 +73,7 @@ On Jan 26, 2026, Andrej Karpathy made [this viral tweet](https://x.com/karpathy/ ![Boris Cherny's response](image.png) -This app template opens that vision up for every Databricks user — no IDE setup, no local installs. Just open the browser and start coding with AI. +This template repo opens that vision up for every Databricks user — no IDE setup, no local installs. Click "Use this template", deploy to Databricks Apps, and start coding with AI in your browser. --- From 2af7c3c7efc62d771e90fc060f8ae1286ac3a77d Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 1 Mar 2026 12:01:06 -0500 Subject: [PATCH 054/382] feat: reinit git on Databricks Apps startup to remove template origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Databricks Apps, the cloned .git/config retains `origin` pointing to the template repo. Users running `git push` from the in-browser terminal could accidentally push to the template instead of their own repo. Adds `_reinit_app_git()` which strips the template's git history and creates a clean, remote-free repo on first startup. Only runs on Databricks Apps (app_dir == /app/python/source_code) — local dev is unaffected. Called after git identity and hooks are written so the initial commit uses the correct author. Also updates README to recommend the "Use this template" workflow for local development, and adds a deployment doc note about the reinit. Co-Authored-By: Claude Opus 4.6 --- README.md | 7 +++++-- app.py | 24 ++++++++++++++++++++++++ docs/deployment.md | 4 +++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dbd76b41..921da0c4 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,12 @@ That's it. Open the app URL and start coding. ### Run locally +1. Click [**Use this template**](https://github.com/datasciencemonkey/coding-agents-databricks-apps/generate) to create your own repo +2. Clone your new repo and run: + ```bash -git clone https://github.com/datasciencemonkey/coding-agents-in-databricks.git -cd coding-agents-in-databricks +git clone https://github.com//.git +cd uv run python app.py ``` diff --git a/app.py b/app.py index dbf9bea4..b9cbfdf2 100644 --- a/app.py +++ b/app.py @@ -167,6 +167,30 @@ def _setup_git_config(): os.chmod(post_commit, 0o755) logger.info(f"Post-commit hook written to {post_commit}") + # Reinit app source git to remove template origin (Databricks Apps only) + _reinit_app_git() + + +def _reinit_app_git(): + """On Databricks Apps, reinit git to remove template origin remote.""" + app_dir = os.path.dirname(os.path.abspath(__file__)) + if app_dir != "/app/python/source_code": + return # Local dev — leave git intact + + git_dir = os.path.join(app_dir, ".git") + if not os.path.isdir(git_dir): + return # Already clean + + import shutil + shutil.rmtree(git_dir) + subprocess.run(["git", "init"], cwd=app_dir, capture_output=True) + subprocess.run(["git", "add", "."], cwd=app_dir, capture_output=True) + subprocess.run( + ["git", "commit", "-m", "Initial commit from coding-agents template"], + cwd=app_dir, capture_output=True, + ) + logger.info("Reinitialized app source git (template origin removed)") + def run_setup(): with setup_lock: diff --git a/docs/deployment.md b/docs/deployment.md index 092b4a4b..8f793fb6 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -19,7 +19,9 @@ The simplest way — no CLI, no cloning, everything stays in the Databricks UI. The app pulls the code directly from Git. To update later, just re-deploy — it picks up the latest from the repo. -> **Optional:** If you use [Databricks AI Gateway](https://docs.databricks.com/aws/en/ai-gateway/), also add `DATABRICKS_GATEWAY_HOST` as a secret or environment variable. Otherwise the app falls back to direct model serving endpoints. +> **Note:** On first startup, the app automatically removes the template's `.git` history and reinitializes a clean, remote-free git repo. This prevents accidental pushes back to the template repo from the in-browser terminal. + +> **Optional (Highly Recommended):** If you use [Databricks AI Gateway](https://docs.databricks.com/aws/en/ai-gateway/), also add `DATABRICKS_GATEWAY_HOST` as a secret or environment variable. Otherwise the app falls back to direct model serving endpoints. ## Alternative: Deploy with CLI From 550b7b48115303011a65c2d7179122212e12abb7 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 1 Mar 2026 12:05:39 -0500 Subject: [PATCH 055/382] test: add test suite for _reinit_app_git 18 tests covering: - Environment detection (skips local dev, runs on Databricks Apps) - Idempotency (safe on restarts when .git already removed) - Git operations (correct commands: init, add, commit) - Call ordering (reinit after identity is written) - Post-commit hook safety (skips app source outside ~/projects/) - .gitignore coverage (excludes __pycache__, .venv, .env, etc.) - Documentation consistency (deployment.md and README updated) Co-Authored-By: Claude Opus 4.6 --- tests/__init__.py | 0 tests/test_reinit_app_git.py | 247 +++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/test_reinit_app_git.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_reinit_app_git.py b/tests/test_reinit_app_git.py new file mode 100644 index 00000000..ae0d7182 --- /dev/null +++ b/tests/test_reinit_app_git.py @@ -0,0 +1,247 @@ +"""Tests for _reinit_app_git() — git reinit on Databricks Apps startup.""" + +import os +import subprocess +import textwrap +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _import_app(): + """Import app module (deferred so monkeypatching can happen first).""" + import app as app_module + return app_module + + +# --------------------------------------------------------------------------- +# 1. Environment detection — only runs on Databricks Apps +# --------------------------------------------------------------------------- + +class TestEnvironmentDetection: + """_reinit_app_git should only act when app_dir == /app/python/source_code.""" + + def test_skips_on_local_dev(self, tmp_path): + """On local dev (any path != /app/python/source_code), function is a no-op.""" + app_mod = _import_app() + # Create a fake .git dir in tmp_path to prove it's NOT touched + fake_git = tmp_path / ".git" + fake_git.mkdir() + + with mock.patch("os.path.abspath", return_value=str(tmp_path / "app.py")): + app_mod._reinit_app_git() + + assert fake_git.is_dir(), ".git should NOT be removed on local dev" + + def test_runs_on_databricks_apps(self, tmp_path): + """When app_dir == /app/python/source_code, reinit should execute.""" + app_mod = _import_app() + fake_git = tmp_path / ".git" + fake_git.mkdir() + + with mock.patch("os.path.abspath", return_value="/app/python/source_code/app.py"), \ + mock.patch("os.path.isdir", return_value=True), \ + mock.patch("shutil.rmtree") as mock_rm, \ + mock.patch("subprocess.run") as mock_run: + app_mod._reinit_app_git() + + mock_rm.assert_called_once_with("/app/python/source_code/.git") + assert mock_run.call_count == 3 # git init, git add, git commit + + +# --------------------------------------------------------------------------- +# 2. Idempotency — safe on restarts +# --------------------------------------------------------------------------- + +class TestIdempotency: + """Function should be safe to call multiple times.""" + + def test_skips_when_git_dir_missing(self): + """If .git already removed (e.g. restart), function is a no-op.""" + app_mod = _import_app() + + with mock.patch("os.path.abspath", return_value="/app/python/source_code/app.py"), \ + mock.patch("os.path.isdir", return_value=False), \ + mock.patch("shutil.rmtree") as mock_rm, \ + mock.patch("subprocess.run") as mock_run: + app_mod._reinit_app_git() + + mock_rm.assert_not_called() + mock_run.assert_not_called() + + +# --------------------------------------------------------------------------- +# 3. Git operations — correct commands issued +# --------------------------------------------------------------------------- + +class TestGitOperations: + """Verify the exact git commands issued during reinit.""" + + def test_removes_git_dir(self): + """Should call shutil.rmtree on the .git directory.""" + app_mod = _import_app() + + with mock.patch("os.path.abspath", return_value="/app/python/source_code/app.py"), \ + mock.patch("os.path.isdir", return_value=True), \ + mock.patch("shutil.rmtree") as mock_rm, \ + mock.patch("subprocess.run"): + app_mod._reinit_app_git() + + mock_rm.assert_called_once_with("/app/python/source_code/.git") + + def test_runs_git_init(self): + """Should run git init in the app directory.""" + app_mod = _import_app() + + with mock.patch("os.path.abspath", return_value="/app/python/source_code/app.py"), \ + mock.patch("os.path.isdir", return_value=True), \ + mock.patch("shutil.rmtree"), \ + mock.patch("subprocess.run") as mock_run: + app_mod._reinit_app_git() + + calls = mock_run.call_args_list + assert calls[0] == mock.call( + ["git", "init"], cwd="/app/python/source_code", capture_output=True + ) + + def test_runs_git_add_all(self): + """Should run git add . in the app directory.""" + app_mod = _import_app() + + with mock.patch("os.path.abspath", return_value="/app/python/source_code/app.py"), \ + mock.patch("os.path.isdir", return_value=True), \ + mock.patch("shutil.rmtree"), \ + mock.patch("subprocess.run") as mock_run: + app_mod._reinit_app_git() + + calls = mock_run.call_args_list + assert calls[1] == mock.call( + ["git", "add", "."], cwd="/app/python/source_code", capture_output=True + ) + + def test_runs_git_commit_with_template_message(self): + """Should run git commit with the template message.""" + app_mod = _import_app() + + with mock.patch("os.path.abspath", return_value="/app/python/source_code/app.py"), \ + mock.patch("os.path.isdir", return_value=True), \ + mock.patch("shutil.rmtree"), \ + mock.patch("subprocess.run") as mock_run: + app_mod._reinit_app_git() + + calls = mock_run.call_args_list + assert calls[2] == mock.call( + ["git", "commit", "-m", "Initial commit from coding-agents template"], + cwd="/app/python/source_code", capture_output=True, + ) + + +# --------------------------------------------------------------------------- +# 4. Call ordering — reinit happens after identity is written +# --------------------------------------------------------------------------- + +class TestCallOrdering: + """_reinit_app_git must be called AFTER git identity is configured.""" + + def test_reinit_called_at_end_of_setup_git_config(self): + """Verify _reinit_app_git() is the last thing _setup_git_config() does.""" + import inspect + app_mod = _import_app() + source = inspect.getsource(app_mod._setup_git_config) + lines = source.strip().split("\n") + + # Find the _reinit_app_git() call + reinit_lines = [i for i, l in enumerate(lines) if "_reinit_app_git()" in l and "def " not in l] + assert reinit_lines, "_reinit_app_git() call not found in _setup_git_config" + + # It should be near the end (last few lines, allowing for comments/whitespace) + last_reinit = reinit_lines[-1] + remaining = [l.strip() for l in lines[last_reinit + 1:] if l.strip() and not l.strip().startswith("#")] + assert remaining == [], f"Code after _reinit_app_git(): {remaining}" + + def test_gitconfig_written_before_reinit(self): + """Verify .gitconfig is written before _reinit_app_git is called.""" + import inspect + app_mod = _import_app() + source = inspect.getsource(app_mod._setup_git_config) + + gitconfig_pos = source.find("Git config written to") + reinit_pos = source.find("_reinit_app_git()") + assert gitconfig_pos < reinit_pos, "gitconfig should be written before reinit is called" + + +# --------------------------------------------------------------------------- +# 5. Post-commit hook safety — hook skips app source +# --------------------------------------------------------------------------- + +class TestPostCommitHookSafety: + """The post-commit hook must not sync app source to workspace.""" + + def test_hook_skips_non_project_repos(self): + """The case statement in the hook should skip repos outside ~/projects/.""" + import inspect + app_mod = _import_app() + source = inspect.getsource(app_mod._setup_git_config) + + # Verify the hook has the PROJECTS_DIR guard + assert 'PROJECTS_DIR="$HOME/projects"' in source + assert 'case "$REPO_ROOT" in' in source + assert '"$PROJECTS_DIR"/*)' in source + assert "exit 0" in source + + def test_app_source_is_outside_projects_dir(self): + """App source (/app/python/source_code) is not inside ~/projects/.""" + app_source = "/app/python/source_code" + projects_dir = "/app/python/source_code/projects" + assert not app_source.startswith(projects_dir + "/"), \ + "App source should NOT be inside projects dir" + + +# --------------------------------------------------------------------------- +# 6. .gitignore coverage +# --------------------------------------------------------------------------- + +class TestGitignore: + """Ensure .gitignore excludes sensitive/generated files from git add .""" + + @pytest.fixture + def gitignore_content(self): + gitignore_path = os.path.join(os.path.dirname(__file__), "..", ".gitignore") + with open(gitignore_path) as f: + return f.read() + + @pytest.mark.parametrize("pattern", [ + "__pycache__/", + "*.pyc", + ".env", + ".venv/", + "venv/", + ]) + def test_gitignore_excludes(self, gitignore_content, pattern): + assert pattern in gitignore_content, f"{pattern} missing from .gitignore" + + +# --------------------------------------------------------------------------- +# 7. Documentation consistency +# --------------------------------------------------------------------------- + +class TestDocumentation: + """Verify docs mention the git reinit behavior.""" + + def test_deployment_docs_mention_reinit(self): + docs_path = os.path.join(os.path.dirname(__file__), "..", "docs", "deployment.md") + with open(docs_path) as f: + content = f.read() + assert "reinitializes" in content or "reinit" in content, \ + "deployment.md should mention git reinit" + + def test_readme_recommends_template(self): + readme_path = os.path.join(os.path.dirname(__file__), "..", "README.md") + with open(readme_path) as f: + content = f.read() + assert "Use this template" in content, \ + "README should recommend 'Use this template' workflow" From 82c2feb3f637b6627b56b43182036e12fc0e2b2d Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 1 Mar 2026 12:59:16 -0500 Subject: [PATCH 056/382] feat: add clipboard paste and drag-and-drop image upload with inline image support Users can now paste (Cmd+V) or drag-and-drop images into the terminal. Images are uploaded to ~/uploads/ via /api/upload and the file path is inserted into the terminal input. Loads @xterm/addon-image for inline Sixel/iTerm image rendering. Text paste continues to work natively. Closes #31 Co-Authored-By: Claude Opus 4.6 --- .gitignore | 6 ++ app.py | 24 ++++++ static/index.html | 89 ++++++++++++++++++++++ static/lib/addon-image.js | 3 + tests/test_upload.py | 154 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 276 insertions(+) create mode 100644 static/lib/addon-image.js create mode 100644 tests/test_upload.py diff --git a/.gitignore b/.gitignore index a6efe312..33ee1c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,9 @@ outstanding-todos.md # Git worktrees .worktrees/ + +# Human tokens (brainstorming notes) +.humantokens/ + +# Uploaded files (clipboard paste images) +uploads/ diff --git a/app.py b/app.py index b9cbfdf2..c1a767ab 100644 --- a/app.py +++ b/app.py @@ -12,6 +12,7 @@ import copy import logging from flask import Flask, send_from_directory, request, jsonify, session +from werkzeug.utils import secure_filename from collections import deque from utils import ensure_https @@ -462,6 +463,29 @@ def send_input(): return jsonify({"error": str(e)}), 500 +@app.route("/api/upload", methods=["POST"]) +def upload_file(): + """Save an uploaded file (e.g. clipboard image) and return its path.""" + if "file" not in request.files: + return jsonify({"error": "No file provided"}), 400 + + f = request.files["file"] + if not f.filename: + return jsonify({"error": "Empty filename"}), 400 + + home = os.environ.get("HOME", "/app/python/source_code") + if not home or home == "/": + home = "/app/python/source_code" + upload_dir = os.path.join(home, "uploads") + os.makedirs(upload_dir, exist_ok=True) + + safe_name = f"{uuid.uuid4().hex[:8]}_{secure_filename(f.filename)}" + file_path = os.path.join(upload_dir, safe_name) + f.save(file_path) + + return jsonify({"path": file_path}) + + @app.route("/api/output", methods=["POST"]) def get_output(): """Get output from the terminal.""" diff --git a/static/index.html b/static/index.html index eaad4354..c3ff8c94 100644 --- a/static/index.html +++ b/static/index.html @@ -257,6 +257,7 @@ + From 5745918af1552cb3e03056d0da3ecc6baf1a948a Mon Sep 17 00:00:00 2001 From: datasciencemonkey Date: Sun, 8 Mar 2026 04:47:52 -0400 Subject: [PATCH 074/382] refactor: introduce tabs array data model Co-Authored-By: Claude Opus 4.6 --- static/index.html | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/static/index.html b/static/index.html index 936f1d69..0c7e2f71 100644 --- a/static/index.html +++ b/static/index.html @@ -416,19 +416,34 @@ let lastDarkTheme = localStorage.getItem('terminal-last-dark') || 'Dark'; let lastLightTheme = localStorage.getItem('terminal-last-light') || 'Light'; - // ── Pane Object Model ───────────────────────────────────────── - // Each pane: { id, element, term, fitAddon, searchAddon, sessionId } - let panes = []; - let activePaneId = null; + // ── Tab & Pane Object Model ─────────────────────────────────── + // Tab: { id, label, panes[], activePaneId, paneContainer, divider } + // Pane: { id, element, term, fitAddon, searchAddon, sessionId } + const MAX_TABS = 5; + let tabs = []; + let activeTabId = null; + let tabIdCounter = 0; let paneIdCounter = 0; + function getActiveTab() { + return tabs.find(t => t.id === activeTabId) || tabs[0]; + } + function getActivePane() { - return panes.find(p => p.id === activePaneId) || panes[0]; + const tab = getActiveTab(); + if (!tab) return null; + return tab.panes.find(p => p.id === tab.activePaneId) || tab.panes[0]; + } + + function getAllPanes() { + return tabs.flatMap(t => t.panes); } function focusPane(id) { - activePaneId = id; - panes.forEach(p => { + const tab = getActiveTab(); + if (!tab) return; + tab.activePaneId = id; + tab.panes.forEach(p => { p.element.classList.toggle('active', p.id === id); if (p.id === id) p.term.focus(); }); From 74464278ad9ce7e10ca5a9318a5966deb636f3ac Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 04:48:37 -0400 Subject: [PATCH 075/382] refactor: update theme/font/refit to use tabs model Co-Authored-By: Claude Opus 4.6 --- static/index.html | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/static/index.html b/static/index.html index 0c7e2f71..c3b65292 100644 --- a/static/index.html +++ b/static/index.html @@ -467,7 +467,7 @@ document.getElementById('search-bar').style.background = overlayBg; document.getElementById('dictation-preview').style.background = overlayBg; // Apply to all panes - panes.forEach(p => { p.term.options.theme = preset.theme; }); + getAllPanes().forEach(p => { p.term.options.theme = preset.theme; }); if (preset.type === 'dark') { lastDarkTheme = name; localStorage.setItem('terminal-last-dark', name); @@ -488,7 +488,7 @@ currentFontSize = Math.max(MIN_FONT_SIZE, Math.min(MAX_FONT_SIZE, size)); localStorage.setItem('terminal-font-size', currentFontSize); updateFontSizeDisplay(); - panes.forEach(p => { p.term.options.fontSize = currentFontSize; }); + getAllPanes().forEach(p => { p.term.options.fontSize = currentFontSize; }); refitAllPanes(); } @@ -498,14 +498,16 @@ if (!family) return; currentFontFamily = name; localStorage.setItem('terminal-font-family', name); - panes.forEach(p => { p.term.options.fontFamily = family; }); + getAllPanes().forEach(p => { p.term.options.fontFamily = family; }); refitAllPanes(); document.getElementById('font-family-select').value = name; } // ── Refit all panes ───────────────────────────────────────────── function refitAllPanes() { - panes.forEach(p => { + const tab = getActiveTab(); + if (!tab) return; + tab.panes.forEach(p => { p.fitAddon.fit(); if (p.sessionId) sendResize(p.term.cols, p.term.rows, p.sessionId); }); From 192f2838af667f4589c0e621b6063098eb49bd8d Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 04:50:09 -0400 Subject: [PATCH 076/382] refactor: createPane now accepts parent tab Co-Authored-By: Claude Opus 4.6 --- static/index.html | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/static/index.html b/static/index.html index c3b65292..a88be18a 100644 --- a/static/index.html +++ b/static/index.html @@ -839,19 +839,20 @@ } // ── Pane Management ──────────────────────────────────────────── - async function createPane() { + async function createPane(tab) { const id = 'pane-' + (++paneIdCounter); - const container = document.getElementById('pane-container'); + const container = tab.paneContainer; const element = document.createElement('div'); element.className = 'pane'; element.id = id; // Add divider before second pane - if (panes.length === 1) { + if (tab.panes.length === 1) { const divider = document.createElement('div'); - divider.id = 'pane-divider'; + divider.className = 'pane-divider'; container.appendChild(divider); - setupDividerDrag(divider); + tab.divider = divider; + setupDividerDrag(divider, tab); } container.appendChild(element); @@ -868,12 +869,12 @@ term.loadAddon(new WebLinksAddon.WebLinksAddon()); let searchAddon = null; - if (typeof SearchAddon !== 'undefined') { + if (typeof SearchAddon \!== 'undefined') { searchAddon = new SearchAddon.SearchAddon(); term.loadAddon(searchAddon); } - if (typeof ImageAddon !== 'undefined' && ImageAddon.ImageAddon) { + if (typeof ImageAddon \!== 'undefined' && ImageAddon.ImageAddon) { term.loadAddon(new ImageAddon.ImageAddon({ sixelSupport: true, sixelScrolling: true, @@ -891,7 +892,7 @@ term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n'); - term.write('\x1b[90mCtrl+Shift+D split pane \u2502 Ctrl+Shift+W close pane \u2502 Ctrl+Shift+] switch pane\x1b[0m\r\n\r\n'); + term.write('\x1b[90mCtrl+Shift+T new tab \u2502 Alt+Shift+D split pane \u2502 Alt+Shift+W close pane\x1b[0m\r\n\r\n'); const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid }; term.onData(data => sendInput(data, pane.sessionId)); @@ -900,7 +901,7 @@ // Click to focus element.addEventListener('mousedown', () => focusPane(id)); - panes.push(pane); + tab.panes.push(pane); focusPane(id); return pane; From 68711a6eff6861f962a80d22e0cd2ae3740f6429 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 04:51:17 -0400 Subject: [PATCH 077/382] feat: implement createTab, switchTab, closeTab, renameTab Co-Authored-By: Claude Opus 4.6 --- static/index.html | 192 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) diff --git a/static/index.html b/static/index.html index a88be18a..f15539bc 100644 --- a/static/index.html +++ b/static/index.html @@ -907,6 +907,198 @@ return pane; } + // ── Tab Management ────────────────────────────────────────────── + async function createTab() { + if (tabs.length >= MAX_TABS) return null; + + const id = 'tab-' + (++tabIdCounter); + const label = 'Shell ' + tabIdCounter; + + // Create per-tab pane container + const paneContainer = document.createElement('div'); + paneContainer.className = 'tab-pane-container'; + paneContainer.id = id + '-panes'; + document.body.appendChild(paneContainer); + + const tab = { + id, + label, + panes: [], + activePaneId: null, + paneContainer, + divider: null + }; + + tabs.push(tab); + + // Render tab in the tab bar + renderTabBar(); + + // Switch to new tab (hides others) + switchTab(id); + + // Create first pane + await createPane(tab); + + updateTabButtons(); + return tab; + } + + function switchTab(id) { + const prevTab = getActiveTab(); + activeTabId = id; + + // Toggle pane container visibility + tabs.forEach(t => { + t.paneContainer.classList.toggle('hidden', t.id !== id); + }); + + // Update tab bar active state + renderTabBar(); + + // Refit panes in the newly visible tab and focus + const tab = getActiveTab(); + if (tab && tab.panes.length > 0) { + requestAnimationFrame(() => { + refitAllPanes(); + const ap = tab.panes.find(p => p.id === tab.activePaneId) || tab.panes[0]; + if (ap) ap.term.focus(); + }); + } + } + + function closeTab(id) { + const tab = tabs.find(t => t.id === id); + if (!tab) return; + + // Cleanup all panes in this tab + tab.panes.forEach(p => { + cleanupPane(p); + p.term.dispose(); + }); + + // Remove DOM + tab.paneContainer.remove(); + + // Remove from array + tabs = tabs.filter(t => t.id !== id); + + // If we closed the active tab, switch to the last tab + if (activeTabId === id) { + if (tabs.length > 0) { + switchTab(tabs[tabs.length - 1].id); + } + } + + // If no tabs left, create a new one + if (tabs.length === 0) { + tabIdCounter = 0; + createTab(); + return; + } + + renderTabBar(); + updateTabButtons(); + } + + function startRenameTab(id) { + const labelEl = document.querySelector(`#tab-bar .tab[data-tab-id="${id}"] .tab-label`); + if (!labelEl) return; + labelEl.contentEditable = 'true'; + labelEl.focus(); + + // Select all text + const range = document.createRange(); + range.selectNodeContents(labelEl); + window.getSelection().removeAllRanges(); + window.getSelection().addRange(range); + + function finishRename() { + labelEl.contentEditable = 'false'; + const newLabel = labelEl.textContent.trim(); + const tab = tabs.find(t => t.id === id); + if (tab && newLabel) { + tab.label = newLabel; + } else if (tab) { + labelEl.textContent = tab.label; // revert empty + } + labelEl.removeEventListener('blur', finishRename); + labelEl.removeEventListener('keydown', handleKey); + // Refocus terminal + const ap = getActivePane(); + if (ap) ap.term.focus(); + } + + function handleKey(e) { + if (e.key === 'Enter') { + e.preventDefault(); + finishRename(); + } + if (e.key === 'Escape') { + e.preventDefault(); + const tab = tabs.find(t => t.id === id); + if (tab) labelEl.textContent = tab.label; + finishRename(); + } + } + + labelEl.addEventListener('blur', finishRename); + labelEl.addEventListener('keydown', handleKey); + } + + function renderTabBar() { + const tabBar = document.getElementById('tab-bar'); + const newTabBtn = document.getElementById('new-tab-btn'); + + // Remove old tab elements (keep the + button) + tabBar.querySelectorAll('.tab').forEach(el => el.remove()); + + // Insert tabs before the + button + tabs.forEach((tab, index) => { + const tabEl = document.createElement('div'); + tabEl.className = 'tab' + (tab.id === activeTabId ? ' active' : ''); + tabEl.dataset.tabId = tab.id; + + const label = document.createElement('span'); + label.className = 'tab-label'; + label.textContent = tab.label; + tabEl.appendChild(label); + + const closeBtn = document.createElement('button'); + closeBtn.className = 'tab-close'; + closeBtn.textContent = '\u00D7'; + closeBtn.title = 'Close tab'; + closeBtn.addEventListener('click', (e) => { + e.stopPropagation(); + closeTab(tab.id); + }); + tabEl.appendChild(closeBtn); + + // Click to switch + tabEl.addEventListener('click', () => switchTab(tab.id)); + + // Double-click to rename + tabEl.addEventListener('dblclick', (e) => { + e.preventDefault(); + startRenameTab(tab.id); + }); + + tabBar.insertBefore(tabEl, newTabBtn); + }); + + // Update + button state + newTabBtn.disabled = tabs.length >= MAX_TABS; + } + + function updateTabButtons() { + // Update toolbar pane buttons for active tab + const tab = getActiveTab(); + const multi = tab && tab.panes.length > 1; + document.getElementById('close-pane-btn').style.display = multi ? '' : 'none'; + document.getElementById('next-pane-btn').style.display = multi ? '' : 'none'; + document.getElementById('split-btn').style.display = (tab && tab.panes.length >= 2) ? 'none' : ''; + } + async function splitPane() { if (panes.length >= 2) return; status.textContent = 'Splitting...'; From 6ca95428d443062222b392330dd1cc3be41c4e63 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 04:52:10 -0400 Subject: [PATCH 078/382] refactor: pane operations now scoped to active tab Co-Authored-By: Claude Opus 4.6 --- static/index.html | 64 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/static/index.html b/static/index.html index f15539bc..6fb177c5 100644 --- a/static/index.html +++ b/static/index.html @@ -1100,15 +1100,16 @@ } async function splitPane() { - if (panes.length >= 2) return; + const tab = getActiveTab(); + if (!tab || tab.panes.length >= 2) return; status.textContent = 'Splitting...'; status.style.display = ''; try { - await createPane(); + await createPane(tab); // Reset flex for even split - panes.forEach(p => { p.element.style.flex = '1'; }); + tab.panes.forEach(p => { p.element.style.flex = '1'; }); refitAllPanes(); - updatePaneButtons(); + updateTabButtons(); status.style.display = 'none'; } catch (e) { status.textContent = 'Split failed: ' + e.message; @@ -1117,8 +1118,16 @@ } function closeActivePane() { - if (panes.length <= 1) return; - const ap = getActivePane(); + const tab = getActiveTab(); + if (!tab) return; + + // If only one pane, close the whole tab + if (tab.panes.length <= 1) { + closeTab(tab.id); + return; + } + + const ap = tab.panes.find(p => p.id === tab.activePaneId) || tab.panes[0]; if (!ap) return; cleanupPane(ap); @@ -1126,28 +1135,47 @@ ap.element.remove(); // Remove divider - const divider = document.getElementById('pane-divider'); - if (divider) divider.remove(); + if (tab.divider) { + tab.divider.remove(); + tab.divider = null; + } - panes = panes.filter(p => p.id !== ap.id); + tab.panes = tab.panes.filter(p => p.id !== ap.id); // Reset remaining pane to full width - if (panes.length === 1) { - panes[0].element.style.flex = '1'; + if (tab.panes.length === 1) { + tab.panes[0].element.style.flex = '1'; } - focusPane(panes[0].id); + focusPane(tab.panes[0].id); refitAllPanes(); - updatePaneButtons(); + updateTabButtons(); } function cyclePaneFocus(direction) { - if (panes.length <= 1) return; - const idx = panes.findIndex(p => p.id === activePaneId); + const tab = getActiveTab(); + if (!tab || tab.panes.length <= 1) return; + const idx = tab.panes.findIndex(p => p.id === tab.activePaneId); const next = direction === 'next' - ? (idx + 1) % panes.length - : (idx - 1 + panes.length) % panes.length; - focusPane(panes[next].id); + ? (idx + 1) % tab.panes.length + : (idx - 1 + tab.panes.length) % tab.panes.length; + focusPane(tab.panes[next].id); + } + + function cycleTabFocus(direction) { + if (tabs.length <= 1) return; + const idx = tabs.findIndex(t => t.id === activeTabId); + const next = direction === 'next' + ? (idx + 1) % tabs.length + : (idx - 1 + tabs.length) % tabs.length; + switchTab(tabs[next].id); + } + + function jumpToTab(number) { + // number is 1-indexed + if (number >= 1 && number <= tabs.length) { + switchTab(tabs[number - 1].id); + } } // ── Divider Drag ─────────────────────────────────────────────── From 2656c78f0e9e54ace18712877d9c0fd0b97ad7bb Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 04:52:44 -0400 Subject: [PATCH 079/382] refactor: divider drag scoped to parent tab Co-Authored-By: Claude Opus 4.6 --- static/index.html | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/static/index.html b/static/index.html index 6fb177c5..299fa112 100644 --- a/static/index.html +++ b/static/index.html @@ -1179,7 +1179,7 @@ } // ── Divider Drag ─────────────────────────────────────────────── - function setupDividerDrag(divider) { + function setupDividerDrag(divider, tab) { let dragging = false; divider.addEventListener('mousedown', e => { @@ -1191,13 +1191,12 @@ }); document.addEventListener('mousemove', e => { - if (!dragging || panes.length < 2) return; - const container = document.getElementById('pane-container'); - const rect = container.getBoundingClientRect(); + if (!dragging || tab.panes.length < 2) return; + const rect = tab.paneContainer.getBoundingClientRect(); let pct = ((e.clientX - rect.left) / rect.width) * 100; pct = Math.max(15, Math.min(85, pct)); - panes[0].element.style.flex = `0 0 ${pct}%`; - panes[1].element.style.flex = '1 1 0'; + tab.panes[0].element.style.flex = `0 0 ${pct}%`; + tab.panes[1].element.style.flex = '1 1 0'; refitAllPanes(); }); From 7003e8f154f35ec367763968d7c02f8acef0ebc0 Mon Sep 17 00:00:00 2001 From: datasciencemonkey Date: Sun, 8 Mar 2026 04:53:40 -0400 Subject: [PATCH 080/382] feat: add tab keyboard shortcuts, move pane shortcuts to Alt+Shift Co-Authored-By: Claude Opus 4.6 --- static/index.html | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/static/index.html b/static/index.html index 299fa112..064d742e 100644 --- a/static/index.html +++ b/static/index.html @@ -270,9 +270,9 @@ 🎤 - - - + + + @@ -724,20 +724,44 @@ else startDictation(); return; } - // Ctrl+Shift+D : split pane - if (e.ctrlKey && e.shiftKey && e.key === 'D') { - e.preventDefault(); splitPane(); return; + + // ── Tab shortcuts (Ctrl+Shift) ── + // Ctrl+Shift+T : new tab + if (e.ctrlKey && e.shiftKey && e.key === 'T') { + e.preventDefault(); createTab(); return; } - // Ctrl+Shift+W : close active pane + // Ctrl+Shift+W : close active pane (closes tab if last pane) if (e.ctrlKey && e.shiftKey && e.key === 'W') { e.preventDefault(); closeActivePane(); return; } - // Ctrl+Shift+] : next pane — use e.code because Shift+] produces '}' + // Ctrl+Shift+] : next tab if (e.ctrlKey && e.shiftKey && e.code === 'BracketRight') { - e.preventDefault(); cyclePaneFocus('next'); return; + e.preventDefault(); cycleTabFocus('next'); return; } - // Ctrl+Shift+[ : prev pane — use e.code because Shift+[ produces '{' + // Ctrl+Shift+[ : prev tab if (e.ctrlKey && e.shiftKey && e.code === 'BracketLeft') { + e.preventDefault(); cycleTabFocus('prev'); return; + } + // Ctrl+Shift+1-5 : jump to tab + if (e.ctrlKey && e.shiftKey && e.code >= 'Digit1' && e.code <= 'Digit5') { + e.preventDefault(); jumpToTab(parseInt(e.code.slice(-1))); return; + } + + // ── Pane shortcuts (Alt+Shift) ── + // Alt+Shift+D : split pane + if (e.altKey && e.shiftKey && e.key === 'D') { + e.preventDefault(); splitPane(); return; + } + // Alt+Shift+W : close pane + if (e.altKey && e.shiftKey && e.key === 'W') { + e.preventDefault(); closeActivePane(); return; + } + // Alt+Shift+] : next pane + if (e.altKey && e.shiftKey && e.code === 'BracketRight') { + e.preventDefault(); cyclePaneFocus('next'); return; + } + // Alt+Shift+[ : prev pane + if (e.altKey && e.shiftKey && e.code === 'BracketLeft') { e.preventDefault(); cyclePaneFocus('prev'); return; } }); From 79df0a9289e982268a885c2d3b9b9605047e1bc0 Mon Sep 17 00:00:00 2001 From: datasciencemonkey Date: Sun, 8 Mar 2026 04:54:34 -0400 Subject: [PATCH 081/382] feat: wire new-tab button, update cleanup for tabs Co-Authored-By: Claude Opus 4.6 --- static/index.html | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/static/index.html b/static/index.html index 064d742e..26bee87f 100644 --- a/static/index.html +++ b/static/index.html @@ -799,7 +799,7 @@ pollWorker.onmessage = function(event) { const msg = event.data; - const pane = panes.find(p => p.id === msg.paneId); + const pane = getAllPanes().find(p => p.id === msg.paneId); if (!pane) return; switch (msg.type) { @@ -840,7 +840,7 @@ // sendBeacon heartbeat on pagehide as safety net before Worker dies window.addEventListener('pagehide', () => { - panes.forEach(p => { + getAllPanes().forEach(p => { if (p.sessionId) { navigator.sendBeacon( '/api/heartbeat', @@ -859,7 +859,7 @@ } function cleanupAllPanes() { - panes.forEach(p => cleanupPane(p)); + getAllPanes().forEach(p => cleanupPane(p)); } // ── Pane Management ──────────────────────────────────────────── @@ -1236,17 +1236,11 @@ } // ── Pane toolbar buttons ──────────────────────────────────────── + document.getElementById('new-tab-btn').addEventListener('click', () => createTab()); document.getElementById('split-btn').addEventListener('click', () => splitPane()); document.getElementById('close-pane-btn').addEventListener('click', () => closeActivePane()); document.getElementById('next-pane-btn').addEventListener('click', () => cyclePaneFocus('next')); - function updatePaneButtons() { - const multi = panes.length > 1; - document.getElementById('close-pane-btn').style.display = multi ? '' : 'none'; - document.getElementById('next-pane-btn').style.display = multi ? '' : 'none'; - document.getElementById('split-btn').style.display = panes.length >= 2 ? 'none' : ''; - } - // ── Toast Notification ────────────────────────────────────────── function showToast(message, type = 'info') { const toast = document.createElement('div'); From 8aab33c2a97aa1715df0bc7f1d81539858bdddb9 Mon Sep 17 00:00:00 2001 From: datasciencemonkey Date: Sun, 8 Mar 2026 04:55:13 -0400 Subject: [PATCH 082/382] feat: init creates first tab, multi-tab terminals complete Co-Authored-By: Claude Opus 4.6 --- static/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index 26bee87f..87441be4 100644 --- a/static/index.html +++ b/static/index.html @@ -1357,7 +1357,7 @@ if (typeof Terminal === 'undefined') throw new Error('xterm.js not loaded'); if (typeof FitAddon === 'undefined') throw new Error('FitAddon not loaded'); - await createPane(); + await createTab(); status.textContent = 'Connected!'; setTimeout(() => { status.style.display = 'none'; }, 1000); From b97b214fb4172fea57291268df3f968ea6feefff Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:01:29 -0400 Subject: [PATCH 083/382] fix: remove escaped backslashes in createPane JS The subagent introduced \\!== instead of \!== in the SearchAddon and ImageAddon type checks, causing a syntax error that prevented init() from running. Co-Authored-By: Claude Opus 4.6 --- static/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/index.html b/static/index.html index 87441be4..f9a6c5ba 100644 --- a/static/index.html +++ b/static/index.html @@ -893,12 +893,12 @@ term.loadAddon(new WebLinksAddon.WebLinksAddon()); let searchAddon = null; - if (typeof SearchAddon \!== 'undefined') { + if (typeof SearchAddon !== 'undefined') { searchAddon = new SearchAddon.SearchAddon(); term.loadAddon(searchAddon); } - if (typeof ImageAddon \!== 'undefined' && ImageAddon.ImageAddon) { + if (typeof ImageAddon !== 'undefined' && ImageAddon.ImageAddon) { term.loadAddon(new ImageAddon.ImageAddon({ sixelSupport: true, sixelScrolling: true, From 157e07defd8391b77b0a04b6c4742f532398e3cd Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:03:15 -0400 Subject: [PATCH 084/382] fix: double-click rename by skipping switchTab on active tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clicking the already-active tab triggered switchTab → renderTabBar, which destroyed and recreated the DOM elements before the dblclick event could fire. Now switchTab returns early if already on that tab. Co-Authored-By: Claude Opus 4.6 --- static/index.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index f9a6c5ba..87cec093 100644 --- a/static/index.html +++ b/static/index.html @@ -969,7 +969,9 @@ } function switchTab(id) { - const prevTab = getActiveTab(); + // Skip if already on this tab (preserves DOM for dblclick rename) + if (activeTabId === id) return; + activeTabId = id; // Toggle pane container visibility From 82dcbca898633a16a8ac596c1d0c0b95b53be6d7 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:05:34 -0400 Subject: [PATCH 085/382] feat: increase max tabs from 5 to 10, extend jump shortcut to 1-9 Co-Authored-By: Claude Opus 4.6 --- static/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/static/index.html b/static/index.html index 87cec093..c521cbcc 100644 --- a/static/index.html +++ b/static/index.html @@ -419,7 +419,7 @@ // ── Tab & Pane Object Model ─────────────────────────────────── // Tab: { id, label, panes[], activePaneId, paneContainer, divider } // Pane: { id, element, term, fitAddon, searchAddon, sessionId } - const MAX_TABS = 5; + const MAX_TABS = 10; let tabs = []; let activeTabId = null; let tabIdCounter = 0; @@ -742,8 +742,8 @@ if (e.ctrlKey && e.shiftKey && e.code === 'BracketLeft') { e.preventDefault(); cycleTabFocus('prev'); return; } - // Ctrl+Shift+1-5 : jump to tab - if (e.ctrlKey && e.shiftKey && e.code >= 'Digit1' && e.code <= 'Digit5') { + // Ctrl+Shift+1-9 : jump to tab + if (e.ctrlKey && e.shiftKey && e.code >= 'Digit1' && e.code <= 'Digit9') { e.preventDefault(); jumpToTab(parseInt(e.code.slice(-1))); return; } From 42bc7aa758e1d7dbc2a0e1837e4cf50b71835f73 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:07:38 -0400 Subject: [PATCH 086/382] fix: use capture phase for keyboard shortcuts to prevent xterm.js swallowing Ctrl+Shift+number was being consumed by xterm.js before our handler. Using capture phase (3rd arg = true) ensures our shortcuts fire first. Co-Authored-By: Claude Opus 4.6 --- static/index.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/static/index.html b/static/index.html index c521cbcc..932ec11e 100644 --- a/static/index.html +++ b/static/index.html @@ -700,6 +700,7 @@ }); // ── Global Keyboard Shortcuts ────────────────────────────────── + // Use capture phase so shortcuts fire before xterm.js swallows the event document.addEventListener('keydown', e => { // Ctrl+= : increase font if (e.ctrlKey && !e.altKey && !e.shiftKey && (e.key === '=' || e.key === '+')) { @@ -744,7 +745,7 @@ } // Ctrl+Shift+1-9 : jump to tab if (e.ctrlKey && e.shiftKey && e.code >= 'Digit1' && e.code <= 'Digit9') { - e.preventDefault(); jumpToTab(parseInt(e.code.slice(-1))); return; + e.preventDefault(); e.stopPropagation(); jumpToTab(parseInt(e.code.slice(-1))); return; } // ── Pane shortcuts (Alt+Shift) ── @@ -764,7 +765,7 @@ if (e.altKey && e.shiftKey && e.code === 'BracketLeft') { e.preventDefault(); cyclePaneFocus('prev'); return; } - }); + }, true); // capture phase — fire before xterm.js // ── Session / IO (parameterized by sessionId) ────────────────── const status = document.getElementById('status'); From d83d88b82566eec7863d3b5ac0cb85bff4572473 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:08:57 -0400 Subject: [PATCH 087/382] fix: use e.code for Alt+Shift shortcuts (macOS Option key changes e.key) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On macOS, Alt+Shift+D produces 'Î' not 'D' for e.key. Using e.code (physical key) makes shortcuts work regardless of OS key remapping. Also added stopPropagation to prevent xterm.js from processing them. Co-Authored-By: Claude Opus 4.6 --- static/index.html | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/static/index.html b/static/index.html index 932ec11e..f0d83506 100644 --- a/static/index.html +++ b/static/index.html @@ -749,21 +749,22 @@ } // ── Pane shortcuts (Alt+Shift) ── + // Use e.code (physical key) because Alt/Option on macOS changes e.key // Alt+Shift+D : split pane - if (e.altKey && e.shiftKey && e.key === 'D') { - e.preventDefault(); splitPane(); return; + if (e.altKey && e.shiftKey && e.code === 'KeyD') { + e.preventDefault(); e.stopPropagation(); splitPane(); return; } // Alt+Shift+W : close pane - if (e.altKey && e.shiftKey && e.key === 'W') { - e.preventDefault(); closeActivePane(); return; + if (e.altKey && e.shiftKey && e.code === 'KeyW') { + e.preventDefault(); e.stopPropagation(); closeActivePane(); return; } // Alt+Shift+] : next pane if (e.altKey && e.shiftKey && e.code === 'BracketRight') { - e.preventDefault(); cyclePaneFocus('next'); return; + e.preventDefault(); e.stopPropagation(); cyclePaneFocus('next'); return; } // Alt+Shift+[ : prev pane if (e.altKey && e.shiftKey && e.code === 'BracketLeft') { - e.preventDefault(); cyclePaneFocus('prev'); return; + e.preventDefault(); e.stopPropagation(); cyclePaneFocus('prev'); return; } }, true); // capture phase — fire before xterm.js From aba8af1c550689f8cd720660a0c5e7ed24856d63 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:11:59 -0400 Subject: [PATCH 088/382] fix: truncate long tab labels with ellipsis, keep close button visible Added overflow:hidden + text-overflow:ellipsis on .tab-label and flex-shrink:0 on .tab-close so the X button never gets pushed out. Co-Authored-By: Claude Opus 4.6 --- static/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index f0d83506..cfd3d025 100644 --- a/static/index.html +++ b/static/index.html @@ -37,6 +37,7 @@ outline: none; border: none; background: none; color: inherit; font: inherit; padding: 0; min-width: 30px; max-width: 120px; + overflow: hidden; text-overflow: ellipsis; cursor: inherit; } .tab-label:focus { @@ -47,7 +48,7 @@ opacity: 0; font-size: 10px; padding: 2px 4px; border-radius: 3px; border: none; background: none; color: inherit; cursor: pointer; transition: opacity 0.15s, background 0.15s; - line-height: 1; + line-height: 1; flex-shrink: 0; } .tab:hover .tab-close, .tab.active .tab-close { opacity: 0.6; } .tab-close:hover { opacity: 1 !important; background: rgba(255,255,255,0.1); } From 1c2456fb9135210d648f6d96bce11665af3ec997 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:33:47 -0400 Subject: [PATCH 089/382] feat: add CoDA ASCII art splash screen on terminal launch Replaces the plain welcome text with a cyan ASCII block-letter "CoDA" logo (with lowercase o), tagline, version indicator, and keyboard shortcut hints. Closes #43 Co-Authored-By: Claude Sonnet 4.6 --- static/index.html | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/static/index.html b/static/index.html index cfd3d025..1ea52656 100644 --- a/static/index.html +++ b/static/index.html @@ -917,9 +917,26 @@ const sid = await createSession(); await sendResize(term.cols, term.rows, sid); - term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); - term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n'); - term.write('\x1b[90mCtrl+Shift+T new tab \u2502 Alt+Shift+D split pane \u2502 Alt+Shift+W close pane\x1b[0m\r\n\r\n'); + // CoDA splash screen (lowercase o) + const splashArt = [ + ' \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 ', + '\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557', + '\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551', + '\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551', + '\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u255a\u2588\u2588\u2588\u2588\u2588\u2554\u255d \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d \u2588\u2588\u2551 \u2588\u2588\u2551', + ' \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u255d', + ]; + term.write('\x1b[2J\x1b[H'); // clear screen, cursor home + term.write('\r\n'); + splashArt.forEach(line => term.write('\x1b[36m' + line + '\x1b[0m\r\n')); + term.write('\r\n'); + term.write('\x1b[1;37m CoWorking Developer Agents\x1b[0m\r\n'); + term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); + term.write('\x1b[90m v0.1.0 \u2502 Ready\x1b[0m\r\n'); + term.write('\r\n'); + term.write('\x1b[90m Ctrl+Shift+T new tab \u2502 Alt+Shift+D split \u2502 Alt+Shift+W close\x1b[0m\r\n'); + term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); + term.write('\r\n'); const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid }; term.onData(data => sendInput(data, pane.sessionId)); From 373d56f2e48b032d023b2c2056eb2064fe285020 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:38:14 -0400 Subject: [PATCH 090/382] revert: remove CoDA splash screen, restore original welcome text Co-Authored-By: Claude Sonnet 4.6 --- static/index.html | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/static/index.html b/static/index.html index 1ea52656..cfd3d025 100644 --- a/static/index.html +++ b/static/index.html @@ -917,26 +917,9 @@ const sid = await createSession(); await sendResize(term.cols, term.rows, sid); - // CoDA splash screen (lowercase o) - const splashArt = [ - ' \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 ', - '\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557', - '\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551', - '\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551', - '\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u255a\u2588\u2588\u2588\u2588\u2588\u2554\u255d \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d \u2588\u2588\u2551 \u2588\u2588\u2551', - ' \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u255d', - ]; - term.write('\x1b[2J\x1b[H'); // clear screen, cursor home - term.write('\r\n'); - splashArt.forEach(line => term.write('\x1b[36m' + line + '\x1b[0m\r\n')); - term.write('\r\n'); - term.write('\x1b[1;37m CoWorking Developer Agents\x1b[0m\r\n'); - term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); - term.write('\x1b[90m v0.1.0 \u2502 Ready\x1b[0m\r\n'); - term.write('\r\n'); - term.write('\x1b[90m Ctrl+Shift+T new tab \u2502 Alt+Shift+D split \u2502 Alt+Shift+W close\x1b[0m\r\n'); - term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); - term.write('\r\n'); + term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); + term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n'); + term.write('\x1b[90mCtrl+Shift+T new tab \u2502 Alt+Shift+D split pane \u2502 Alt+Shift+W close pane\x1b[0m\r\n\r\n'); const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid }; term.onData(data => sendInput(data, pane.sessionId)); From 9ca5cb9b60dd38847e58988858e2464e8e33832d Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:39:23 -0400 Subject: [PATCH 091/382] Revert "revert: remove CoDA splash screen, restore original welcome text" This reverts commit 373d56f2e48b032d023b2c2056eb2064fe285020. --- static/index.html | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/static/index.html b/static/index.html index cfd3d025..88344930 100644 --- a/static/index.html +++ b/static/index.html @@ -917,9 +917,26 @@ const sid = await createSession(); await sendResize(term.cols, term.rows, sid); - term.write('\x1b[32mConnected. Type "claude" to start coding.\x1b[0m\r\n'); - term.write('\x1b[90mProjects in ~/projects auto-sync to Workspace on git commit.\x1b[0m\r\n'); - term.write('\x1b[90mCtrl+Shift+T new tab \u2502 Alt+Shift+D split pane \u2502 Alt+Shift+W close pane\x1b[0m\r\n\r\n'); + // CoDA splash screen + const splashArt = [ + ' \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 ', + '\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557', + '\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551', + '\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551', + '\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u2588\u2588\u2551', + ' \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u255d', + ]; + term.write('\x1b[2J\x1b[H'); // clear screen, cursor home + term.write('\r\n'); + splashArt.forEach(line => term.write('\x1b[36m' + line + '\x1b[0m\r\n')); + term.write('\r\n'); + term.write('\x1b[1;37m CoWorking Developer Agents\x1b[0m\r\n'); + term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); + term.write('\x1b[90m v0.1.0 \u2502 Ready\x1b[0m\r\n'); + term.write('\r\n'); + term.write('\x1b[90m Ctrl+Shift+T new tab \u2502 Alt+Shift+D split \u2502 Alt+Shift+W close\x1b[0m\r\n'); + term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); + term.write('\r\n'); const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid }; term.onData(data => sendInput(data, pane.sessionId)); From f89ae4e1c7ae92c91446facc22d7ff67b036cd2e Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:46:13 -0400 Subject: [PATCH 092/382] feat: add keyboard shortcuts help popup (closes #44) Adds a "?" button in the toolbar and Ctrl+/ keybinding to toggle a modal overlay showing all keyboard shortcuts grouped by category. Dismissible via Escape, click-outside, or the shortcut toggle. Co-Authored-By: Claude Sonnet 4.6 --- static/index.html | 96 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/static/index.html b/static/index.html index 88344930..8e590dc9 100644 --- a/static/index.html +++ b/static/index.html @@ -167,6 +167,44 @@ animation: pulse 1s infinite; flex-shrink: 0; } + /* Shortcuts help modal */ + #shortcuts-overlay { + display: none; position: fixed; inset: 0; z-index: 2000; + background: rgba(0,0,0,0.5); backdrop-filter: blur(4px); + align-items: center; justify-content: center; + } + #shortcuts-overlay.visible { display: flex; } + #shortcuts-modal { + width: min(90vw, 460px); max-height: 80vh; overflow-y: auto; + border-radius: 14px; padding: 24px; + border: 1px solid rgba(255,255,255,0.1); + backdrop-filter: blur(24px); -webkit-backdrop-filter: blur(24px); + box-shadow: 0 16px 48px rgba(0,0,0,0.4); + } + #shortcuts-modal h2 { + margin: 0 0 16px; font-size: 16px; font-weight: 600; + display: flex; align-items: center; justify-content: space-between; + } + #shortcuts-modal h2 button { + background: none; border: none; color: inherit; cursor: pointer; + font-size: 18px; opacity: 0.5; padding: 2px 6px; border-radius: 4px; + } + #shortcuts-modal h2 button:hover { opacity: 1; background: rgba(255,255,255,0.1); } + #shortcuts-modal h3 { + margin: 14px 0 6px; font-size: 11px; text-transform: uppercase; + letter-spacing: 1.5px; opacity: 0.4; font-weight: 600; + } + #shortcuts-modal h3:first-of-type { margin-top: 0; } + #shortcuts-modal .shortcut-row { + display: flex; justify-content: space-between; align-items: center; + padding: 5px 0; font-size: 13px; + } + #shortcuts-modal .shortcut-row span:last-child { + font-family: monospace; font-size: 11px; opacity: 0.6; + background: rgba(255,255,255,0.06); padding: 2px 8px; + border-radius: 4px; border: 1px solid rgba(255,255,255,0.08); + } + /* Search bar */ #search-bar { display: none; position: fixed; top: 10px; right: 40px; z-index: 1001; @@ -274,6 +312,7 @@ + @@ -302,6 +341,31 @@ + +
+
+

Keyboard Shortcuts

+

Tabs

+
New tabCtrl+Shift+T
+
Close tabCtrl+Shift+W
+
Next tabCtrl+Shift+]
+
Previous tabCtrl+Shift+[
+
Jump to tab 1-9Ctrl+Shift+1-9
+

Panes

+
Split paneAlt+Shift+D
+
Close paneAlt+Shift+W
+
Next paneAlt+Shift+]
+
Previous paneAlt+Shift+[
+

General

+
SearchCtrl+Shift+F
+
Voice dictationAlt+V
+
Increase fontCtrl+=
+
Decrease fontCtrl+-
+
Reset fontCtrl+0
+
This helpCtrl+/
+
+
+
@@ -602,6 +666,30 @@ if (e.key === 'Escape') { e.preventDefault(); toggleSearch(); } }); + // ── Shortcuts Help Modal ──────────────────────────────────────── + const shortcutsOverlay = document.getElementById('shortcuts-overlay'); + const shortcutsModal = document.getElementById('shortcuts-modal'); + + function toggleShortcutsHelp() { + const visible = shortcutsOverlay.classList.toggle('visible'); + if (!visible) { + const ap = getActivePane(); + if (ap) ap.term.focus(); + } + // Theme-aware background + const preset = themes[currentThemeName]; + if (preset) { + shortcutsModal.style.background = preset.type === 'dark' + ? 'rgba(30,30,30,0.95)' : 'rgba(245,245,245,0.95)'; + } + } + + document.getElementById('shortcuts-btn').addEventListener('click', toggleShortcutsHelp); + document.getElementById('shortcuts-close').addEventListener('click', toggleShortcutsHelp); + shortcutsOverlay.addEventListener('click', (e) => { + if (e.target === shortcutsOverlay) toggleShortcutsHelp(); + }); + // ── Voice Dictation ──────────────────────────────────────────── const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; let recognition = null; @@ -715,6 +803,14 @@ if (e.ctrlKey && !e.altKey && !e.shiftKey && e.key === '0') { e.preventDefault(); setFontSize(DEFAULT_FONT_SIZE); return; } + // Ctrl+/ : toggle shortcuts help + if (e.ctrlKey && !e.altKey && !e.shiftKey && e.key === '/') { + e.preventDefault(); toggleShortcutsHelp(); return; + } + // Escape : close shortcuts help if open + if (e.key === 'Escape' && shortcutsOverlay.classList.contains('visible')) { + e.preventDefault(); e.stopPropagation(); toggleShortcutsHelp(); return; + } // Ctrl+Shift+F : toggle search if (e.ctrlKey && e.shiftKey && e.key === 'F') { e.preventDefault(); toggleSearch(); return; From 1eabee53996c53e3d851c345f3936e251c2853a0 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:47:36 -0400 Subject: [PATCH 093/382] refactor: replace inline shortcuts with Ctrl+/ hint in splash screen Now that the shortcuts help popup exists, point users there instead of listing individual shortcuts in the splash screen. Co-Authored-By: Claude Sonnet 4.6 --- static/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index 8e590dc9..193f7715 100644 --- a/static/index.html +++ b/static/index.html @@ -1030,7 +1030,7 @@

General

term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); term.write('\x1b[90m v0.1.0 \u2502 Ready\x1b[0m\r\n'); term.write('\r\n'); - term.write('\x1b[90m Ctrl+Shift+T new tab \u2502 Alt+Shift+D split \u2502 Alt+Shift+W close\x1b[0m\r\n'); + term.write('\x1b[90m Ctrl+/ for keyboard shortcuts\x1b[0m\r\n'); term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); term.write('\r\n'); From 54d719724d279cc5f84e3805a2d1f6cd3ea87951 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:51:34 -0400 Subject: [PATCH 094/382] feat: add version.json as single source of truth for app version - version.json at project root (readable by backend, frontend, CI/CD) - Backend loads version at startup, exposes /api/version endpoint - Health endpoint now includes version - Frontend fetches version from /api/version for splash screen Co-Authored-By: Claude Sonnet 4.6 --- app.py | 16 ++++++++++++++++ static/index.html | 9 ++++++++- version.json | 1 + 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 version.json diff --git a/app.py b/app.py index 278af44a..d042b3a5 100644 --- a/app.py +++ b/app.py @@ -15,8 +15,18 @@ from werkzeug.utils import secure_filename from collections import deque +import json as _json + from utils import ensure_https +# App version (single source of truth: version.json) +_version_file = os.path.join(os.path.dirname(__file__), 'version.json') +try: + with open(_version_file) as _f: + APP_VERSION = _json.load(_f).get('version', '0.0.0') +except Exception: + APP_VERSION = '0.0.0' + # Session timeout configuration SESSION_TIMEOUT_SECONDS = 300 # No poll for 5 min = dead session CLEANUP_INTERVAL_SECONDS = 60 # How often to check for stale sessions @@ -401,12 +411,18 @@ def health(): current_setup_status = setup_state["status"] return jsonify({ "status": "healthy", + "version": APP_VERSION, "setup_status": current_setup_status, "active_sessions": session_count, "session_timeout_seconds": SESSION_TIMEOUT_SECONDS }) +@app.route("/api/version") +def get_version(): + return jsonify({"version": APP_VERSION}) + + @app.route("/api/session", methods=["POST"]) def create_session(): """Create a new terminal session.""" diff --git a/static/index.html b/static/index.html index 193f7715..5efe0432 100644 --- a/static/index.html +++ b/static/index.html @@ -1028,7 +1028,7 @@

General

term.write('\r\n'); term.write('\x1b[1;37m CoWorking Developer Agents\x1b[0m\r\n'); term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); - term.write('\x1b[90m v0.1.0 \u2502 Ready\x1b[0m\r\n'); + term.write('\x1b[90m v' + appVersion + ' \u2502 Ready\x1b[0m\r\n'); term.write('\r\n'); term.write('\x1b[90m Ctrl+/ for keyboard shortcuts\x1b[0m\r\n'); term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); @@ -1467,6 +1467,13 @@

General

} }); + // ── Version ────────────────────────────────────────────────────── + let appVersion = '0.0.0'; + try { + const vResp = await fetch('/api/version'); + if (vResp.ok) { const vData = await vResp.json(); appVersion = vData.version || appVersion; } + } catch(e) { /* fallback */ } + // ── Init ─────────────────────────────────────────────────────── async function init() { try { diff --git a/version.json b/version.json new file mode 100644 index 00000000..fb75183f --- /dev/null +++ b/version.json @@ -0,0 +1 @@ +{ "version": "0.1.0" } From 49082edab8664495530818a311f303b6b2a5e53d Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:53:02 -0400 Subject: [PATCH 095/382] refactor: move version to pyproject.toml, remove version.json Standard Python project convention. Backend reads version via tomllib. Frontend still fetches from /api/version. CI can read with: python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])" Co-Authored-By: Claude Sonnet 4.6 --- app.py | 10 +++++----- pyproject.toml | 12 ++++++++++++ version.json | 1 - 3 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 pyproject.toml delete mode 100644 version.json diff --git a/app.py b/app.py index d042b3a5..e4e58c10 100644 --- a/app.py +++ b/app.py @@ -15,15 +15,15 @@ from werkzeug.utils import secure_filename from collections import deque -import json as _json +import tomllib from utils import ensure_https -# App version (single source of truth: version.json) -_version_file = os.path.join(os.path.dirname(__file__), 'version.json') +# App version (single source of truth: pyproject.toml) +_pyproject_file = os.path.join(os.path.dirname(__file__), 'pyproject.toml') try: - with open(_version_file) as _f: - APP_VERSION = _json.load(_f).get('version', '0.0.0') + with open(_pyproject_file, 'rb') as _f: + APP_VERSION = tomllib.load(_f)['project']['version'] except Exception: APP_VERSION = '0.0.0' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..618d13a6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "coda" +version = "0.1.0" +description = "CoWorking Developer Agents — browser-based terminal for Databricks" +requires-python = ">=3.10" +dependencies = [ + "flask>=2.0", + "claude-agent-sdk", + "databricks-sdk>=0.20.0", + "mlflow[genai]>=3.4", + "opentelemetry-exporter-otlp-proto-grpc", +] diff --git a/version.json b/version.json deleted file mode 100644 index fb75183f..00000000 --- a/version.json +++ /dev/null @@ -1 +0,0 @@ -{ "version": "0.1.0" } From 99abd39a7fc566b06e75a6d83fdd8f55db98c3ad Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 05:55:44 -0400 Subject: [PATCH 096/382] feat: add GitHub Actions release workflow, bump to v0.15.0 - Manual workflow_dispatch to create tagged releases from pyproject.toml - Auto-generates categorized release notes from conventional commits - Guards against duplicate tags and invalid semver - Update project description to "CoDA - Coding Agents on Databricks Apps" Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/release.yml | 120 ++++++++++++++++++++++++++++++++++ pyproject.toml | 4 +- 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..e257ca9b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,120 @@ +name: Release + +on: + workflow_dispatch: + inputs: + prerelease: + description: "Mark as pre-release?" + required: false + default: false + type: boolean + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Read version from pyproject.toml + id: version + run: | + VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "Error: version '$VERSION' in pyproject.toml is not valid semver" + exit 1 + fi + echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" + echo "TAG=v$VERSION" >> "$GITHUB_OUTPUT" + echo "Releasing $VERSION" + + - name: Check tag does not already exist + run: | + TAG="${{ steps.version.outputs.TAG }}" + if git tag -l "$TAG" | grep -q "$TAG"; then + echo "Error: tag $TAG already exists — did you forget to bump the version in pyproject.toml?" + exit 1 + fi + + - name: Generate release notes + id: notes + run: | + TAG="${{ steps.version.outputs.TAG }}" + PREV_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "") + + if [ -z "$PREV_TAG" ]; then + RANGE="HEAD" + SINCE_MSG="(all commits — first release)" + else + RANGE="${PREV_TAG}..HEAD" + SINCE_MSG="since $PREV_TAG" + fi + + echo "Generating notes for $RANGE" + + # Categorize commits by conventional commit prefix + FEATURES=$(git log $RANGE --pretty=format:"- %s (%h)" | grep -E "^- feat" | sed 's/^- feat[^:]*: /- /' || true) + FIXES=$(git log $RANGE --pretty=format:"- %s (%h)" | grep -E "^- fix" | sed 's/^- fix[^:]*: /- /' || true) + DOCS=$(git log $RANGE --pretty=format:"- %s (%h)" | grep -E "^- docs" | sed 's/^- docs[^:]*: /- /' || true) + REFACTORS=$(git log $RANGE --pretty=format:"- %s (%h)" | grep -E "^- refactor" | sed 's/^- refactor[^:]*: /- /' || true) + OTHER=$(git log $RANGE --pretty=format:"- %s (%h)" | grep -vE "^- (feat|fix|docs|refactor|chore|ci|test|style|perf|revert|Merge)" || true) + + { + echo "NOTES<> "$GITHUB_OUTPUT" + + - name: Create and push tag + run: | + TAG="${{ steps.version.outputs.TAG }}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: "${{ steps.version.outputs.TAG }}" + name: "${{ steps.version.outputs.TAG }}" + body: ${{ steps.notes.outputs.NOTES }} + prerelease: ${{ inputs.prerelease }} diff --git a/pyproject.toml b/pyproject.toml index 618d13a6..c4559dfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "coda" -version = "0.1.0" -description = "CoWorking Developer Agents — browser-based terminal for Databricks" +version = "0.15.0" +description = "CoDA - Coding Agents on Databricks Apps" requires-python = ">=3.10" dependencies = [ "flask>=2.0", From 33ff388fed2cabf3e88460a5752d806df0fab05f Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 07:58:26 -0400 Subject: [PATCH 097/382] feat: batch polling, security headers, SIGTERM fix (no tmux) - Add /api/output-batch endpoint for multi-pane polling efficiency - Rewrite poll-worker.js to use batch polling (1 request per cycle) - Add security response headers (X-Content-Type-Options, etc.) - Fix SIGTERM handler: register only in gunicorn, not at module level - Fix top-level await bug in frontend init() - Worker retries on shutting_down instead of dying permanently - Document tmux evaluation and decision to not adopt See docs/2026-03-08-tmux-evaluation.md for full analysis. Co-Authored-By: dgokeeffe Co-Authored-By: Claude Opus 4.6 --- app.py | 67 +++++++- docs/2026-03-08-tmux-evaluation.md | 95 +++++++++++ static/index.html | 10 +- static/poll-worker.js | 245 ++++++++++++++--------------- 4 files changed, 281 insertions(+), 136 deletions(-) create mode 100644 docs/2026-03-08-tmux-evaluation.md diff --git a/app.py b/app.py index e4e58c10..9bd38e1e 100644 --- a/app.py +++ b/app.py @@ -46,13 +46,21 @@ # SIGTERM graceful shutdown: notify clients before gunicorn stops the worker shutting_down = False +_start_time = time.time() + def handle_sigterm(signum, frame): """Notify clients that app is shutting down, then let gunicorn handle the rest.""" global shutting_down + # Ignore SIGTERMs in the first 10s — likely stale signals from a prior process kill + if time.time() - _start_time < 10: + logger.info("SIGTERM received during startup — ignoring (likely stale signal)") + return shutting_down = True logger.info("SIGTERM received — setting shutting_down flag for clients") -signal.signal(signal.SIGTERM, handle_sigterm) +# NOTE: Do not register SIGTERM handler at module level. +# It is installed in initialize_app() for gunicorn only. +# For local dev (__main__), we keep SIG_DFL so the process just exits. # Setup state tracking setup_lock = threading.Lock() @@ -389,6 +397,15 @@ def authorize_request(): return None +@app.after_request +def set_security_headers(response): + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["X-XSS-Protection"] = "1; mode=block" + response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" + return response + + @app.route("/") def index(): with setup_lock: @@ -546,6 +563,42 @@ def get_output(): return jsonify({"output": output, "exited": exited, "shutting_down": shutting_down, "timeout_warning": timeout_warning}) +@app.route("/api/output-batch", methods=["POST"]) +def get_output_batch(): + """Get output from multiple terminal sessions in one request. + + Accepts: {"session_ids": ["id1", "id2", ...]} + Returns: {"outputs": {"id1": {"output": "...", "exited": false}, ...}} + """ + data = request.json or {} + session_ids = data.get("session_ids") + + if session_ids is None: + return jsonify({"error": "session_ids required"}), 400 + + outputs = {} + now = time.time() + + with sessions_lock: + for sid in session_ids: + if sid not in sessions: + continue + session = sessions[sid] + session["last_poll_time"] = now + buffer = session["output_buffer"] + output = "".join(buffer) + buffer.clear() + exited = session.get("exited", False) + timeout_warning = session.pop("timeout_warning", False) + outputs[sid] = { + "output": output, + "exited": exited, + "timeout_warning": timeout_warning + } + + return jsonify({"outputs": outputs, "shutting_down": shutting_down}) + + @app.route("/api/heartbeat", methods=["POST"]) def heartbeat(): """Lightweight keep-alive — resets timeout without draining output buffer.""" @@ -603,10 +656,15 @@ def close_session(): return jsonify({"status": "ok"}) -def initialize_app(): +def initialize_app(local_dev=False): """One-time init: detect owner, start cleanup thread.""" global app_owner + # Install SIGTERM handler only for gunicorn (production). + # For local dev, SIG_DFL is fine — the process just exits cleanly. + if not local_dev: + signal.signal(signal.SIGTERM, handle_sigterm) + # Remove OAuth credentials - force PAT auth only os.environ.pop("DATABRICKS_CLIENT_ID", None) os.environ.pop("DATABRICKS_CLIENT_SECRET", None) @@ -631,7 +689,8 @@ def initialize_app(): if __name__ == "__main__": - # Local dev only — production uses gunicorn - initialize_app() + # Local dev — no SIGTERM handler (SIG_DFL), no shutting_down flag + initialize_app(local_dev=True) + shutting_down = False # safety net: ensure clean state before serving port = int(os.environ.get("DATABRICKS_APP_PORT", 8000)) app.run(host="0.0.0.0", port=port, threaded=True) diff --git a/docs/2026-03-08-tmux-evaluation.md b/docs/2026-03-08-tmux-evaluation.md new file mode 100644 index 00000000..9f27225b --- /dev/null +++ b/docs/2026-03-08-tmux-evaluation.md @@ -0,0 +1,95 @@ +# Tmux Evaluation for Session Persistence + +**Date:** 2026-03-08 +**Branch:** feat/multi-tab-terminals +**Verdict:** Remove tmux. Use localStorage-based session recovery instead. + +## What We Tried + +Wrapped PTY sessions in `tmux new-session -A -s pane-{id}` so terminals survive Flask worker restarts and browser tab closes. Added `/api/tmux-sessions` endpoint, tmux AppImage install step, and frontend reattach logic. + +## Problems Observed + +1. **Green status bar** — tmux renders its own status line (`[pane-0] 0:zsh*`), stealing screen space and looking foreign inside xterm.js +2. **Dot fill pattern** — tmux fills unused area with dots when its window size doesn't match the attached client +3. **Resize escape codes leaking** — `^[[8;51;148t` sequences visible in terminal output +4. **Splash screen broken** — tmux reattach suppressed the welcome/coda screen +5. **Double resize management** — xterm.js resizes the PTY, but tmux has its own window size logic, causing conflicts +6. **Keybinding conflicts** — tmux's prefix key (Ctrl-B) can interfere with CLI tools like Claude Code + +## Why Tmux Doesn't Solve the Real Problem + +The main persistence need is surviving **Databricks Apps container restarts**. On container restart: +- All processes die, including the tmux server +- The filesystem is recreated from the deployment artifact +- Only `/Workspace/` files survive + +Tmux helps in two scenarios where the **container stays alive but sessions disconnect**: + +1. **Browser tab close/refresh** — user accidentally closes tab while Claude Code is mid-task +2. **Gunicorn worker restart** — `timeout = 30` in gunicorn.conf.py means any request >30s causes gunicorn to SIGKILL the worker and spawn a new one. This is NOT rare during heavy setup or long-running requests. + +**Risk of removing tmux:** In scenario 1, without tmux (or an equivalent), a running Claude Code session is orphaned and killed after the session timeout. This is a real user pain point — losing a 10-minute coding task because of an accidental tab close. + +**Mitigation:** localStorage-based session recovery (see below) addresses scenario 1 without tmux's visual baggage. Scenario 2 remains a gap — if gunicorn kills the worker, PTY FDs are gone and no application-level trick can recover them. Tmux genuinely solves this; our approach does not. + +**Accepted risk:** We accept the gunicorn worker restart gap because (a) it requires a request to exceed 30s which is uncommon during normal terminal use, and (b) the visual/UX cost of tmux outweighs the protection it provides for this edge case. + +## What About David's state_sync.py? + +`state_sync.py` persists two things to `/Workspace/Users/{email}/.state/` every 5 min: +1. `~/.claude/projects/*/memory/` — Claude Code auto-memory +2. `~/.bash_history` — shell history + +### Honest assessment + +**Claude auto-memory IS valuable.** CLAUDE.md covers project-level instructions, but auto-memory accumulates session-specific learnings: "tried approach X, failed because Y", user preferences discovered during conversation, debugging insights. These can't be replicated by CLAUDE.md alone and are lost on every container restart without state_sync. + +**Shell history has low value** in this context — users mostly interact via AI agents, not manual shell commands. + +**Verdict on state_sync:** Worth adopting in a future PR for the auto-memory persistence alone. Not blocking for the current multi-tab work, but genuinely useful. We were too dismissive initially. + +## What Already Works (and Gaps) + +- **Post-commit hook** (`sync_to_workspace.py`): syncs `~/projects/*` repos to Workspace on every git commit. **Gap:** only committed code survives — uncommitted WIP is lost. +- **`GIT_REPOS` env var**: auto-clones repos on startup, so code is restored +- **Web Worker polling**: handles browser background/foreground transitions without session loss +- **5-minute session timeout**: keeps orphaned PTY sessions alive long enough for tab-refresh reconnection. **Gap:** if user steps away longer than 5 min, session is killed. + +## Recommended Approach: localStorage Session Recovery + +Instead of tmux, store session IDs in `localStorage`. On page load: + +``` +1. Check localStorage for previous session_id +2. POST /api/output with old session_id +3. If responds → reattach xterm.js to existing PTY +4. If 404 → create new session +``` + +Benefits: +- Running processes survive tab close/refresh (same as tmux) +- No visual artifacts (no status bar, no dot fill, no resize conflicts) +- Splash screen works normally on new sessions +- Zero extra dependencies + +Trade-offs vs tmux: +- **No scrollback replay** — xterm.js buffer is lost on refresh, user sees blank terminal attached to a running process. Tmux replays the visible screen (~24-50 rows), which is meaningfully better. +- **No gunicorn worker crash recovery** — if gunicorn kills the worker, PTY FDs are gone. Tmux survives this; localStorage recovery does not. +- **Simpler, lighter, no visual bugs** — the trade-off we're choosing to make. + +## What to Keep from feat/multi-tab-terminals + +- `/api/output-batch` endpoint — single request for N panes instead of N requests +- Web Worker batch polling rewrite — background-throttle-immune +- Security response headers (X-Content-Type-Options, X-Frame-Options, etc.) +- SIGTERM handler fix (don't register at module level, only in gunicorn) + +## What to Strip + +- tmux session wrapping in `create_session()` +- `/api/tmux-sessions` endpoint +- tmux install step in `run_setup()` +- `checkTmuxSessions()` in frontend +- Reattach/splash suppression logic +- `pane_id` parameter (only needed for tmux session naming) diff --git a/static/index.html b/static/index.html index 5efe0432..5a6d00e6 100644 --- a/static/index.html +++ b/static/index.html @@ -1469,14 +1469,16 @@

General

// ── Version ────────────────────────────────────────────────────── let appVersion = '0.0.0'; - try { - const vResp = await fetch('/api/version'); - if (vResp.ok) { const vData = await vResp.json(); appVersion = vData.version || appVersion; } - } catch(e) { /* fallback */ } // ── Init ─────────────────────────────────────────────────────── async function init() { try { + // Fetch version inside async init() to avoid top-level await + try { + const vResp = await fetch('/api/version'); + if (vResp.ok) { const vData = await vResp.json(); appVersion = vData.version || appVersion; } + } catch(e) { /* fallback */ } + status.textContent = 'Initializing terminal...'; if (typeof Terminal === 'undefined') throw new Error('xterm.js not loaded'); diff --git a/static/poll-worker.js b/static/poll-worker.js index 7dc1675e..ca31b6fd 100644 --- a/static/poll-worker.js +++ b/static/poll-worker.js @@ -2,8 +2,8 @@ * poll-worker.js — Web Worker for terminal output polling and heartbeat. * * Runs in a Web Worker so it is NOT throttled by the browser when the tab - * is in the background. Manages per-pane polling state, retry/backoff, - * and foreground/background mode switching. + * is in the background. Uses batch polling to fetch output for all panes + * in a single HTTP request. * * Message protocol (main → worker): * { type: 'start_poll', paneId, sessionId } @@ -12,7 +12,7 @@ * * Message protocol (worker → main): * { type: 'output', paneId, data } - * { type: 'session_ended', paneId, reason } — 'exited' | 'auth_expired' | 'shutting_down' + * { type: 'session_ended', paneId, reason } * { type: 'connection_status', paneId, status, attempt, maxAttempts } * { type: 'session_dead', paneId } */ @@ -21,7 +21,7 @@ "use strict"; // ── Constants ───────────────────────────────────────────────────────────── -const POLL_INTERVAL_FG = 100; // ms — foreground output poll +const POLL_INTERVAL_FG = 100; // ms — foreground batch poll const HEARTBEAT_INTERVAL_BG = 30000; // ms — background heartbeat const RETRY_BASE_MS = 500; const RETRY_MULTIPLIER = 2; @@ -30,196 +30,186 @@ const RETRY_MAX_ATTEMPTS = 5; // ── Per-pane state ──────────────────────────────────────────────────────── const panes = new Map(); -// Each entry: { sessionId, pollTimerId, heartbeatTimerId, retryCount, mode } +// Each entry: { sessionId } -let globalHidden = false; // current tab visibility +let globalHidden = false; +let batchTimerId = null; +let retryCount = 0; // ── Retry helpers ───────────────────────────────────────────────────────── function retryDelay(attempt) { const base = RETRY_BASE_MS * Math.pow(RETRY_MULTIPLIER, attempt); const capped = Math.min(base, RETRY_MAX_DELAY_MS); - // Add jitter: 0.5x–1.5x return capped * (0.5 + Math.random()); } -// ── Polling logic ───────────────────────────────────────────────────────── +// ── Batch polling logic ────────────────────────────────────────────────── -async function pollOutput(paneId) { - const state = panes.get(paneId); - if (!state) return; +async function batchPoll() { + if (panes.size === 0) return; + + const sessionIds = []; + const sidToPaneId = new Map(); + for (const [paneId, state] of panes) { + sessionIds.push(state.sessionId); + sidToPaneId.set(state.sessionId, paneId); + } try { - const resp = await fetch("/api/output", { + const resp = await fetch("/api/output-batch", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ session_id: state.sessionId }), + body: JSON.stringify({ session_ids: sessionIds }), }); if (!resp.ok) { if (resp.status === 403) { - self.postMessage({ type: "session_ended", paneId, reason: "auth_expired" }); - stopPane(paneId); + for (const paneId of panes.keys()) { + self.postMessage({ type: "session_ended", paneId, reason: "auth_expired" }); + } + stopAllPanes(); return; } - // 404 or 5xx — retryable throw new Error(`HTTP ${resp.status}`); } - // Success — reset retry counter - state.retryCount = 0; + retryCount = 0; + const result = await resp.json(); - const data = await resp.json(); - - if (data.shutting_down) { - self.postMessage({ type: "session_ended", paneId, reason: "shutting_down" }); - stopPane(paneId); + if (result.shutting_down) { + for (const paneId of panes.keys()) { + self.postMessage({ type: "session_ended", paneId, reason: "shutting_down" }); + } + // Don't stopAllPanes() — retry with backoff so we + // auto-recover when the new server comes up. + handleRetry(new Error("Server shutting down")); return; } - // Forward output + flags to main thread - self.postMessage({ type: "output", paneId, data }); + // Distribute outputs to each pane + for (const [sid, data] of Object.entries(result.outputs || {})) { + const paneId = sidToPaneId.get(sid); + if (!paneId) continue; + + self.postMessage({ type: "output", paneId, data }); - if (data.exited) { - self.postMessage({ type: "session_ended", paneId, reason: "exited" }); - stopPane(paneId); + if (data.exited) { + self.postMessage({ type: "session_ended", paneId, reason: "exited" }); + panes.delete(paneId); + } } } catch (err) { - handleRetry(paneId, err); + handleRetry(err); } } -async function sendHeartbeat(paneId) { - const state = panes.get(paneId); - if (!state) return; +async function batchHeartbeat() { + if (panes.size === 0) return; + + const sessionIds = []; + for (const state of panes.values()) { + sessionIds.push(state.sessionId); + } try { - const resp = await fetch("/api/heartbeat", { + const resp = await fetch("/api/output-batch", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ session_id: state.sessionId }), + body: JSON.stringify({ session_ids: sessionIds }), }); if (!resp.ok) { if (resp.status === 403) { - self.postMessage({ type: "session_ended", paneId, reason: "auth_expired" }); - stopPane(paneId); + for (const paneId of panes.keys()) { + self.postMessage({ type: "session_ended", paneId, reason: "auth_expired" }); + } + stopAllPanes(); return; } throw new Error(`HTTP ${resp.status}`); } - // Success — reset retry counter - state.retryCount = 0; - - const data = await resp.json(); - if (data.timeout_warning) { - self.postMessage({ - type: "output", - paneId, - data: { timeout_warning: true, output: "", exited: false, shutting_down: false }, - }); + retryCount = 0; + + const result = await resp.json(); + for (const [sid, data] of Object.entries(result.outputs || {})) { + if (data.timeout_warning) { + for (const [paneId, state] of panes) { + if (state.sessionId === sid) { + self.postMessage({ + type: "output", paneId, + data: { timeout_warning: true, output: "", exited: false, shutting_down: false }, + }); + } + } + } } } catch (err) { - handleRetry(paneId, err); + handleRetry(err); } } // ── Retry / backoff ─────────────────────────────────────────────────────── -function handleRetry(paneId, err) { - const state = panes.get(paneId); - if (!state) return; +function handleRetry(err) { + retryCount++; - state.retryCount++; - - if (state.retryCount > RETRY_MAX_ATTEMPTS) { - self.postMessage({ type: "session_dead", paneId }); - stopPane(paneId); + if (retryCount > RETRY_MAX_ATTEMPTS) { + for (const paneId of panes.keys()) { + self.postMessage({ type: "session_dead", paneId }); + } + stopAllPanes(); return; } - // Notify main thread of reconnection attempt - self.postMessage({ - type: "connection_status", - paneId, - status: "reconnecting", - attempt: state.retryCount, - maxAttempts: RETRY_MAX_ATTEMPTS, - }); - - // Stop current timers and schedule retry - clearTimers(state); - const delay = retryDelay(state.retryCount - 1); - state.pollTimerId = setTimeout(() => { - if (!panes.has(paneId)) return; - // Re-notify connected on success (handled in poll/heartbeat success path) + for (const paneId of panes.keys()) { self.postMessage({ - type: "connection_status", - paneId, - status: "connected", - attempt: 0, - maxAttempts: RETRY_MAX_ATTEMPTS, + type: "connection_status", paneId, + status: "reconnecting", + attempt: retryCount, maxAttempts: RETRY_MAX_ATTEMPTS, }); - applyMode(paneId); + } + + clearBatchTimer(); + const delay = retryDelay(retryCount - 1); + batchTimerId = setTimeout(() => { + for (const paneId of panes.keys()) { + self.postMessage({ + type: "connection_status", paneId, + status: "connected", attempt: 0, maxAttempts: RETRY_MAX_ATTEMPTS, + }); + } + startBatchTimer(); }, delay); } -// ── Mode management ─────────────────────────────────────────────────────── +// ── Timer management ────────────────────────────────────────────────────── -function clearTimers(state) { - if (state.pollTimerId) { - clearInterval(state.pollTimerId); - clearTimeout(state.pollTimerId); - state.pollTimerId = null; - } - if (state.heartbeatTimerId) { - clearInterval(state.heartbeatTimerId); - clearTimeout(state.heartbeatTimerId); - state.heartbeatTimerId = null; +function clearBatchTimer() { + if (batchTimerId) { + clearInterval(batchTimerId); + clearTimeout(batchTimerId); + batchTimerId = null; } } -function applyMode(paneId) { - const state = panes.get(paneId); - if (!state) return; - - clearTimers(state); - state.mode = globalHidden ? "background" : "foreground"; +function startBatchTimer() { + clearBatchTimer(); + if (panes.size === 0) return; - if (state.mode === "foreground") { - // Poll output at 100ms - pollOutput(paneId); // immediate first poll - state.pollTimerId = setInterval(() => pollOutput(paneId), POLL_INTERVAL_FG); + if (globalHidden) { + batchHeartbeat(); + batchTimerId = setInterval(() => batchHeartbeat(), HEARTBEAT_INTERVAL_BG); } else { - // Background: heartbeat only at 30s - sendHeartbeat(paneId); // immediate first heartbeat - state.heartbeatTimerId = setInterval(() => sendHeartbeat(paneId), HEARTBEAT_INTERVAL_BG); + batchPoll(); + batchTimerId = setInterval(() => batchPoll(), POLL_INTERVAL_FG); } } -// ── Pane lifecycle ──────────────────────────────────────────────────────── - -function startPane(paneId, sessionId) { - // Stop existing if any - stopPane(paneId); - - const state = { - sessionId, - pollTimerId: null, - heartbeatTimerId: null, - retryCount: 0, - mode: globalHidden ? "background" : "foreground", - }; - panes.set(paneId, state); - applyMode(paneId); -} - -function stopPane(paneId) { - const state = panes.get(paneId); - if (!state) return; - clearTimers(state); - panes.delete(paneId); +function stopAllPanes() { + clearBatchTimer(); + panes.clear(); } // ── Message handler ─────────────────────────────────────────────────────── @@ -229,19 +219,18 @@ self.onmessage = function (event) { switch (msg.type) { case "start_poll": - startPane(msg.paneId, msg.sessionId); + panes.set(msg.paneId, { sessionId: msg.sessionId }); + startBatchTimer(); break; case "stop_poll": - stopPane(msg.paneId); + panes.delete(msg.paneId); + if (panes.size === 0) clearBatchTimer(); break; case "visibility_change": globalHidden = msg.hidden; - // Switch all panes to new mode - for (const paneId of panes.keys()) { - applyMode(paneId); - } + startBatchTimer(); break; } }; From ad9629e7f3267c175e23847c4ae457abc323fa45 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 08:14:23 -0400 Subject: [PATCH 098/382] fix: suppress connection banner for transient network blips Add silent retry threshold (5 consecutive failures) before showing "Connection lost" banner. Transient blips on Databricks Apps proxy/LB are retried silently with 500ms delay instead of alarming the user. Co-Authored-By: Claude Sonnet 4.6 --- static/poll-worker.js | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/static/poll-worker.js b/static/poll-worker.js index ca31b6fd..a499b0a5 100644 --- a/static/poll-worker.js +++ b/static/poll-worker.js @@ -26,7 +26,8 @@ const HEARTBEAT_INTERVAL_BG = 30000; // ms — background heartbeat const RETRY_BASE_MS = 500; const RETRY_MULTIPLIER = 2; const RETRY_MAX_DELAY_MS = 10000; -const RETRY_MAX_ATTEMPTS = 5; +const RETRY_MAX_ATTEMPTS = 8; +const SILENT_RETRY_THRESHOLD = 5; // Don't show banner until this many consecutive failures // ── Per-pane state ──────────────────────────────────────────────────────── const panes = new Map(); @@ -163,23 +164,33 @@ function handleRetry(err) { return; } - for (const paneId of panes.keys()) { - self.postMessage({ - type: "connection_status", paneId, - status: "reconnecting", - attempt: retryCount, maxAttempts: RETRY_MAX_ATTEMPTS, - }); - } - - clearBatchTimer(); - const delay = retryDelay(retryCount - 1); - batchTimerId = setTimeout(() => { + // Only notify the UI after SILENT_RETRY_THRESHOLD consecutive failures. + // Transient blips (1-2 failures) are retried silently. + if (retryCount >= SILENT_RETRY_THRESHOLD) { + const visibleAttempt = retryCount - SILENT_RETRY_THRESHOLD + 1; + const visibleMax = RETRY_MAX_ATTEMPTS - SILENT_RETRY_THRESHOLD + 1; for (const paneId of panes.keys()) { self.postMessage({ type: "connection_status", paneId, - status: "connected", attempt: 0, maxAttempts: RETRY_MAX_ATTEMPTS, + status: "reconnecting", + attempt: visibleAttempt, maxAttempts: visibleMax, }); } + } + + clearBatchTimer(); + const delay = retryCount < SILENT_RETRY_THRESHOLD + ? RETRY_BASE_MS // Quick silent retry for transient failures + : retryDelay(retryCount - SILENT_RETRY_THRESHOLD); + batchTimerId = setTimeout(() => { + if (retryCount >= SILENT_RETRY_THRESHOLD) { + for (const paneId of panes.keys()) { + self.postMessage({ + type: "connection_status", paneId, + status: "connected", attempt: 0, maxAttempts: RETRY_MAX_ATTEMPTS, + }); + } + } startBatchTimer(); }, delay); } From 821d9f486cb4dd35422bf9078865ccdc707f1ba0 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 8 Mar 2026 08:21:54 -0400 Subject: [PATCH 099/382] fix: prevent terminal cutoff in Databricks Apps iframe Use percentage-based heights instead of viewport units (100vh/100vw) so the terminal fits within the Databricks Apps iframe container without overflowing at the bottom. Co-Authored-By: Claude Sonnet 4.6 --- static/index.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/static/index.html b/static/index.html index 5a6d00e6..90d50f78 100644 --- a/static/index.html +++ b/static/index.html @@ -8,12 +8,13 @@ + + + + + + + + + + + + + + + + + +
+ +
+ +
+

CoDA

+

Co-Working Developer Agents

+ +

+ Four AI coding agents. + One Databricks App. Three steps to running. +

+

+ Claude Code, Codex, Gemini CLI, and OpenCode — configured for Unity Catalog, AI Gateway, and Workspace files out of the box. +

+ +
+ + Fork on GitHub + + + See how it works + +
+ + +
+
+
+
+
+
+ + + + +
+
+
+
~/workspace/project
+
+ +
+
+
+
+ +
+ + + + +
+
+
+

Four Agents, One Terminal

+

Pick the right model for the job

+

Different models see different things. Switch agents with a click — they share the same workspace, the same data, the same Databricks context.

+
+ +
+ +
+
+
+ +
+
+

Claude Code

+

Anthropic

+
+
+

databricks-claude-opus-4-6

+

Deep Databricks skills + useful MCP servers. The most deeply integrated agent.

+
+ + +
+
+
+ +
+
+

Codex

+

OpenAI

+
+
+

databricks-codex

+

OpenAI's reasoning engine. Excels at multi-step code generation and refactoring.

+
+ + +
+
+
+ +
+
+

Gemini CLI

+

Google

+
+
+

databricks-gemini-2.5-pro

+

Google's multimodal agent. Vision, long context, and deep reasoning.

+
+ + +
+
+
+ +
+
+

OpenCode

+

Open Source

+
+
+

multi-provider

+

Open-source, multi-provider. Use any model, any backend, full transparency.

+
+
+
+
+ +
+ + + + +
+
+
+

What Ships in the Box

+

Everything is wired together

+

Skills, servers, and integrations keep growing. Here's what's configured today.

+
+ +
+
+
+ +
+

Databricks Skills

+

Pipelines, dashboards, Unity Catalog, Lakebase — a growing library.

+
+ +
+
+ +
+

MCP Servers

+

DeepWiki, Exa, and more — wired into every agent and growing.

+
+ +
+
+ +
+

MLflow Tracing

+

Every agent session auto-traced, queryable via Genie.

+
+ +
+
+ +
+

Workspace Sync

+

git commit auto-pushes to your Workspace path.

+
+ +
+
+ +
+

AI Gateway Routing

+

One config, any model, full cost tracking.

+
+ +
+
+ +
+

Terminal Themes

+

Dracula, Nord, Monokai, and more. Pick your vibe.

+
+ +
+
+ +
+

Voice & Image Input

+

Dictate or drag-drop images into the terminal.

+
+ +
+
+ +
+

Supply Chain Security

+

All deps SHA-pinned. Weekly CVE audits via GitHub Actions.

+
+
+
+
+ +
+ + + + +
+
+
+

Why Databricks Apps?

+

You bring the code.
Databricks brings the infra.

+

Running coding agents locally means juggling API keys, model access, and governance. Databricks Apps handles all of that.

+
+ +
+
+
+ +
+

Identity & Auth

+

+ Your workspace token flows through. No API key juggling. Single-user isolation by default. +

+
+ +
+
+ +
+

AI Gateway

+

+ Route agents to any foundation model — Claude, GPT, Gemini — through one gateway. Usage tracked, costs governed. +

+
+ +
+
+ +
+

Data & Governance

+

+ Unity Catalog, MLflow, Workspace files — agents have native access to your entire lakehouse. +

+
+
+ +
+

+ Databricks Apps gives coding agents what they actually need: identity, models, data, and governance. CoDA just wires it all together. +

+
+
+
+ +
+ + + + +
+
+
+
+
+ +
+
+

Need something more specialized?

+

CoDA agents are general-purpose. Genie Code is bespoke.

+

+ The agents in CoDA — Claude Code, Codex, Gemini CLI, OpenCode — are general-purpose coding agents that work across any codebase. They're great for broad software engineering tasks. +

+

+ But if you need an agent that deeply understands your lakehouse — your table schemas, column lineage, governance policies, pipeline failures — Genie Code is purpose-built for that. It's Databricks' autonomous AI agent for data engineering, data science, and ML work, with native Unity Catalog context that general-purpose agents can't match. +

+ + Read the Genie Code announcement + +
+
+
+
+
+ +
+ + + + +
+
+
+

Get Started

+

Three steps. No Terraform.

+
+ +
+
+
1
+
+

Fork the template

+

One click on GitHub. You get the full CoDA setup — agents, skills, MCP servers, themes, and CI.

+
+
+ +
+
2
+
+

Create a Databricks App

+

Connect your repo, pick a name. Databricks handles compute, networking, and identity.

+
+
+ +
+
3
+
+

Set your token, deploy

+

Add your Databricks token as a secret, hit deploy. Agents start with the app.

+
+
+
+ +
+
+ Dockerfile + Terraform + Kubernetes + app.yaml +
+ + Fork on GitHub + +
+
+
+ +
+ + + + + + + + + + + + From ea25749e62be037b7cca1db1fd38c7765f5e54fb Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Thu, 12 Mar 2026 13:39:36 -0400 Subject: [PATCH 123/382] chore: move landing page to docs/site/index.html Co-authored-by: Isaac --- docs/{ => site}/index.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{ => site}/index.html (100%) diff --git a/docs/index.html b/docs/site/index.html similarity index 100% rename from docs/index.html rename to docs/site/index.html From f39ba1fb9e9b135b53f6a18b4e3cd9df0fe2b246 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Thu, 12 Mar 2026 13:47:30 -0400 Subject: [PATCH 124/382] =?UTF-8?q?chore:=20harden=20supply=20chain=20?= =?UTF-8?q?=E2=80=94=20hashes,=20SHA=20pins,=20weekly=20audit=20(#63)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: harden supply chain — hashes, SHA pins, weekly audit - Regenerate requirements.lock with --generate-hashes (3,057 SHA-256 entries) - Pin all GitHub Actions to immutable commit SHAs (checkout, setup-python, action-gh-release) - Pin pip-audit and uv to exact versions in CI - Add weekly cron schedule to dependency-audit workflow - Add npm package version check step to CI - Add pyproject.toml to audit trigger paths - Update lockfile freshness check to use --generate-hashes - Change Gemini CLI fallback from @nightly to @latest - Add dependabot.yml for automated weekly dependency updates Co-Authored-By: Claude Opus 4.6 (1M context) * chore: bump version to 0.16.4 Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add --no-deps to pip-audit for hash-mode compatibility pip-audit auto-enables --require-hashes when it detects hashes in the lockfile, but sqlalchemy's conditional dependency (greenlet) isn't in the lockfile (platform-conditional). --no-deps tells pip-audit to audit only the explicitly listed packages, which is correct since the lockfile already contains all transitive deps. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: strip hashes before pip-audit to avoid greenlet resolution failure pip-audit's pip backend auto-enables --require-hashes when it sees hash entries, then fails on sqlalchemy's platform-conditional greenlet dep (present on x86_64 CI runners but absent from lockfile compiled on aarch64). Fix: strip --hash lines before auditing. The hashes are verified at install time via pip --require-hashes, not at audit time. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .github/dependabot.yml | 10 + .github/workflows/dependency-audit.yml | 30 +- .github/workflows/release.yml | 4 +- pyproject.toml | 2 +- requirements.lock | 3349 ++++++++++++++++++++++-- setup_gemini.py | 2 +- 6 files changed, 3239 insertions(+), 158 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..cbd920f6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index 21333e46..f4746bee 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -5,32 +5,40 @@ on: paths: - "requirements.txt" - "requirements.lock" + - "pyproject.toml" push: branches: [main] paths: - "requirements.txt" - "requirements.lock" + - "pyproject.toml" + schedule: + - cron: '0 6 * * 1' # Weekly Monday 6am UTC — catch newly disclosed CVEs jobs: audit: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" - - name: Install pip-audit - run: pip install pip-audit + - name: Install audit tools + run: pip install pip-audit==2.9.0 uv==0.7.12 - name: Audit pinned dependencies run: | if [ -f requirements.lock ]; then echo "Auditing requirements.lock (pinned)..." - pip-audit -r requirements.lock --desc on + # Strip hashes before auditing — pip-audit's pip backend chokes on + # platform-conditional deps (greenlet) missing from the lockfile. + # The hashes are verified at install time, not audit time. + sed '/^[[:space:]]*--hash/d' requirements.lock > /tmp/requirements.lock.nohash + pip-audit -r /tmp/requirements.lock.nohash --desc on else echo "::warning::No requirements.lock found — auditing requirements.txt (unpinned)" pip-audit -r requirements.txt --desc on @@ -38,8 +46,14 @@ jobs: - name: Check lockfile is up to date run: | - pip install uv - uv pip compile requirements.txt -o /tmp/requirements.lock.check + uv pip compile requirements.txt -o /tmp/requirements.lock.check --generate-hashes if ! diff -q requirements.lock /tmp/requirements.lock.check > /dev/null 2>&1; then - echo "::warning::requirements.lock is out of date. Run: uv pip compile requirements.txt -o requirements.lock" + echo "::warning::requirements.lock is out of date. Run: uv pip compile requirements.txt -o requirements.lock --generate-hashes" fi + + - name: Audit npm packages + run: | + for pkg in opencode-ai @ai-sdk/openai @openai/codex @google/gemini-cli; do + echo "--- Checking $pkg ---" + npm view "$pkg" version 2>/dev/null || echo "::warning::Could not resolve $pkg" + done diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e257ca9b..4bc53104 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 @@ -112,7 +112,7 @@ jobs: git push origin "$TAG" - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 with: tag_name: "${{ steps.version.outputs.TAG }}" name: "${{ steps.version.outputs.TAG }}" diff --git a/pyproject.toml b/pyproject.toml index 1f19dcaf..9407e4ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coda" -version = "0.16.3" +version = "0.16.4" description = "CoDA - Coding Agents on Databricks Apps" requires-python = ">=3.10" dependencies = [ diff --git a/requirements.lock b/requirements.lock index 1def9ed8..4ef87d4f 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,22 +1,154 @@ # This file was autogenerated by uv via the following command: -# uv pip compile requirements.txt -o requirements.lock -aiohappyeyeballs==2.6.1 +# uv pip compile requirements.txt -o requirements.lock --generate-hashes +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via aiohttp -aiohttp==3.13.3 +aiohttp==3.13.3 \ + --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf \ + --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c \ + --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c \ + --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 \ + --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f \ + --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 \ + --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 \ + --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf \ + --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 \ + --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 \ + --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 \ + --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 \ + --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d \ + --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea \ + --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 \ + --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 \ + --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 \ + --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 \ + --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 \ + --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 \ + --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a \ + --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e \ + --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 \ + --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 \ + --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 \ + --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd \ + --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a \ + --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 \ + --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 \ + --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 \ + --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f \ + --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 \ + --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 \ + --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 \ + --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 \ + --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce \ + --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 \ + --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 \ + --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 \ + --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 \ + --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b \ + --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 \ + --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c \ + --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 \ + --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 \ + --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f \ + --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b \ + --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e \ + --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 \ + --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 \ + --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 \ + --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d \ + --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 \ + --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 \ + --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce \ + --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 \ + --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 \ + --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c \ + --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf \ + --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 \ + --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 \ + --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e \ + --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 \ + --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a \ + --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 \ + --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a \ + --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 \ + --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 \ + --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba \ + --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df \ + --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 \ + --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 \ + --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 \ + --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 \ + --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc \ + --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 \ + --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 \ + --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 \ + --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 \ + --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 \ + --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 \ + --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 \ + --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 \ + --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 \ + --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 \ + --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc \ + --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 \ + --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 \ + --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 \ + --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 \ + --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 \ + --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 \ + --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c \ + --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 \ + --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 \ + --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a \ + --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 \ + --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 \ + --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 \ + --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d \ + --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 \ + --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d \ + --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f \ + --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f \ + --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 \ + --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 \ + --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 \ + --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 \ + --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af \ + --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 \ + --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 \ + --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 \ + --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c \ + --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 \ + --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 \ + --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 \ + --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 \ + --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 \ + --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa \ + --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 # via # litellm # mlflow -aiosignal==1.4.0 +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 # via aiohttp -alembic==1.18.4 +alembic==1.18.4 \ + --hash=sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a \ + --hash=sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc # via mlflow -annotated-doc==0.0.4 +annotated-doc==0.0.4 \ + --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \ + --hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4 # via # fastapi # typer -annotated-types==0.7.0 +annotated-types==0.7.0 \ + --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ + --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 # via pydantic -anyio==4.12.1 +anyio==4.12.1 \ + --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ + --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c # via # claude-agent-sdk # httpx @@ -25,39 +157,257 @@ anyio==4.12.1 # sse-starlette # starlette # watchfiles -attrs==25.4.0 +attrs==25.4.0 \ + --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ + --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 # via # aiohttp # jsonschema # referencing -bidict==0.23.1 +bidict==0.23.1 \ + --hash=sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71 \ + --hash=sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5 # via python-socketio -blinker==1.9.0 +blinker==1.9.0 \ + --hash=sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf \ + --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc # via # flask # flask-socketio -boto3==1.42.66 +boto3==1.42.66 \ + --hash=sha256:3bec5300fb2429c3be8e8961fdb1f11e85195922c8a980022332c20af05616d5 \ + --hash=sha256:7c6c60dc5500e8a2967a306372a5fdb4c7f9a5b8adc5eb9aa2ebb5081c51ff47 # via mlflow -botocore==1.42.66 +botocore==1.42.66 \ + --hash=sha256:39756a21142b646de552d798dde2105759b0b8fa0d881a34c26d15bd4c9448fa \ + --hash=sha256:ac48af1ab527dfa08c4617c387413ca56a7f87780d7bfc1da34ef847a59219a5 # via # boto3 # s3transfer -cachetools==7.0.5 +cachetools==7.0.5 \ + --hash=sha256:0cd042c24377200c1dcd225f8b7b12b0ca53cc2c961b43757e774ebe190fd990 \ + --hash=sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114 # via # mlflow-skinny # mlflow-tracing -certifi==2026.2.25 +certifi==2026.2.25 \ + --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ + --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 # via # httpcore # httpx # requests -cffi==2.0.0 +cffi==2.0.0 \ + --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ + --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ + --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ + --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ + --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ + --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ + --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ + --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ + --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ + --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ + --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ + --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ + --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ + --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ + --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ + --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ + --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ + --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ + --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ + --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ + --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ + --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ + --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ + --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ + --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ + --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ + --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ + --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ + --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ + --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ + --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ + --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ + --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ + --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ + --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ + --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ + --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ + --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ + --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ + --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ + --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ + --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ + --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ + --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ + --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ + --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ + --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ + --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ + --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ + --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ + --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ + --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ + --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ + --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ + --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ + --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ + --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ + --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ + --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ + --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ + --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ + --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ + --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ + --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ + --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ + --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ + --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ + --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ + --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ + --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ + --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ + --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ + --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ + --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ + --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ + --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ + --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ + --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ + --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ + --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ + --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ + --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ + --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ + --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf # via cryptography -charset-normalizer==3.4.5 +charset-normalizer==3.4.5 \ + --hash=sha256:014837af6fabf57121b6254fa8ade10dceabc3528b27b721a64bbc7b8b1d4eb4 \ + --hash=sha256:01a1ed54b953303ca7e310fafe0fe347aab348bd81834a0bcd602eb538f89d66 \ + --hash=sha256:0294916d6ccf2d069727d65973c3a1ca477d68708db25fd758dd28b0827cff54 \ + --hash=sha256:02a9d1b01c1e12c27883b0c9349e0bcd9ae92e727ff1a277207e1a262b1cbf05 \ + --hash=sha256:036c079aa08a6a592b82487f97c60b439428320ed1b2ea0b3912e99d30c77765 \ + --hash=sha256:039215608ac7b358c4da0191d10fc76868567fbf276d54c14721bdedeb6de064 \ + --hash=sha256:0625665e4ebdddb553ab185de5db7054393af8879fb0c87bd5690d14379d6819 \ + --hash=sha256:0a45e504f5e1be0bd385935a8e1507c442349ca36f511a47057a71c9d1d6ea9e \ + --hash=sha256:0b362bcd27819f9c07cbf23db4e0e8cd4b44c5ecd900c2ff907b2b92274a7412 \ + --hash=sha256:0c300cefd9b0970381a46394902cd18eaf2aa00163f999590ace991989dcd0fc \ + --hash=sha256:1088345bcc93c58d8d8f3d783eca4a6e7a7752bbff26c3eee7e73c597c191c2e \ + --hash=sha256:10b473fc8dca1c3ad8559985794815f06ca3fc71942c969129070f2c3cdf7281 \ + --hash=sha256:131716d6786ad5e3dc542f5cc6f397ba3339dc0fb87f87ac30e550e8987756af \ + --hash=sha256:14498a429321de554b140013142abe7608f9d8ccc04d7baf2ad60498374aefa2 \ + --hash=sha256:149ec69866c3d6c2fb6f758dbc014ecb09f30b35a5ca90b6a8a2d4e54e18fdfe \ + --hash=sha256:165c7b21d19365464e8f70e5ce5e12524c58b48c78c1f5a57524603c1ab003f8 \ + --hash=sha256:1827734a5b308b65ac54e86a618de66f935a4f63a8a462ff1e19a6788d6c2262 \ + --hash=sha256:19092dde50335accf365cce21998a1c6dd8eafd42c7b226eb54b2747cdce2fac \ + --hash=sha256:1a374cc0b88aa710e8865dc1bd6edb3743c59f27830f0293ab101e4cf3ce9f85 \ + --hash=sha256:1d1401945cb77787dbd3af2446ff2d75912327c4c3a1526ab7955ecf8600687c \ + --hash=sha256:1f2da5cbb9becfcd607757a169e38fb82aa5fd86fae6653dea716e7b613fe2cf \ + --hash=sha256:259cd1ca995ad525f638e131dbcc2353a586564c038fc548a3fe450a91882139 \ + --hash=sha256:2820a98460c83663dd8ec015d9ddfd1e4879f12e06bb7d0500f044fb477d2770 \ + --hash=sha256:28269983f25a4da0425743d0d257a2d6921ea7d9b83599d4039486ec5b9f911d \ + --hash=sha256:2b970382e4a36bed897c19f310f31d7d13489c11b4f468ddfba42d41cddfb918 \ + --hash=sha256:2da4eedcb6338e2321e831a0165759c0c620e37f8cd044a263ff67493be8ffb3 \ + --hash=sha256:30987f4a8ed169983f93e1be8ffeea5214a779e27ed0b059835c7afe96550ad7 \ + --hash=sha256:30a2b1a48478c3428d047ed9690d57c23038dac838a87ad624c85c0a78ebeb39 \ + --hash=sha256:340810d34ef83af92148e96e3e44cb2d3f910d2bf95e5618a5c467d9f102231d \ + --hash=sha256:3f64c6bf8f32f9133b668c7f7a7cbdbc453412bc95ecdbd157f3b1e377a92990 \ + --hash=sha256:4167a621a9a1a986c73777dbc15d4b5eac8ac5c10393374109a343d4013ec765 \ + --hash=sha256:4354e401eb6dab9aed3c7b4030514328a6c748d05e1c3e19175008ca7de84fb1 \ + --hash=sha256:4481e6da1830c8a1cc0b746b47f603b653dadb690bcd851d039ffaefe70533aa \ + --hash=sha256:4b8551b6e6531e156db71193771c93bda78ffc4d1e6372517fe58ad3b91e4659 \ + --hash=sha256:4cd966c2559f501c6fd69294d082c2934c8dd4719deb32c22961a5ac6db0df1d \ + --hash=sha256:50bcbca6603c06a1dcc7b056ed45c37715fb5d2768feb3bcd37d2313c587a5b9 \ + --hash=sha256:530beedcec9b6e027e7a4b6ce26eed36678aa39e17da85e6e03d7bd9e8e9d7c9 \ + --hash=sha256:568e3c34b58422075a1b49575a6abc616d9751b4d61b23f712e12ebb78fe47b2 \ + --hash=sha256:573ef5814c4b7c0d59a7710aa920eaaaef383bd71626aa420fba27b5cab92e8d \ + --hash=sha256:58ad8270cfa5d4bef1bc85bd387217e14ff154d6630e976c6f56f9a040757475 \ + --hash=sha256:597d10dec876923e5c59e48dbd366e852eacb2b806029491d307daea6b917d7c \ + --hash=sha256:5bcb3227c3d9aaf73eaaab1db7ccd80a8995c509ee9941e2aae060ca6e4e5d81 \ + --hash=sha256:5cffde4032a197bd3b42fd0b9509ec60fb70918d6970e4cc773f20fc9180ca67 \ + --hash=sha256:5fea359734b140d0d6741189fea5478c6091b54ffc69d7ce119e0a05637d8c99 \ + --hash=sha256:60d68e820af339df4ae8358c7a2e7596badeb61e544438e489035f9fbf3246a5 \ + --hash=sha256:610f72c0ee565dfb8ae1241b666119582fdbfe7c0975c175be719f940e110694 \ + --hash=sha256:65a126fb4b070d05340a84fc709dd9e7c75d9b063b610ece8a60197a291d0adf \ + --hash=sha256:65b3c403a5b6b8034b655e7385de4f72b7b244869a22b32d4030b99a60593eca \ + --hash=sha256:66dee73039277eb35380d1b82cccc69cc82b13a66f9f4a18da32d573acf02b7c \ + --hash=sha256:708c7acde173eedd4bfa4028484426ba689d2103b28588c513b9db2cd5ecde9c \ + --hash=sha256:728c6a963dfab66ef865f49286e45239384249672cd598576765acc2a640a636 \ + --hash=sha256:754f96058e61a5e22e91483f823e07df16416ce76afa4ebf306f8e1d1296d43f \ + --hash=sha256:75dfd1afe0b1647449e852f4fb428195a7ed0588947218f7ba929f6538487f02 \ + --hash=sha256:75ee9c1cce2911581a70a3c0919d8bccf5b1cbc9b0e5171400ec736b4b569497 \ + --hash=sha256:76a9d0de4d0eab387822e7b35d8f89367dd237c72e82ab42b9f7bf5e15ada00f \ + --hash=sha256:77be992288f720306ab4108fe5c74797de327f3248368dfc7e1a916d6ed9e5a2 \ + --hash=sha256:7ad83b8f9379176c841f8865884f3514d905bcd2a9a3b210eaa446e7d2223e4d \ + --hash=sha256:8197abe5ca1ffb7d91e78360f915eef5addff270f8a71c1fc5be24a56f3e4873 \ + --hash=sha256:82cc7c2ad42faec8b574351f8bc2a0c049043893853317bd9bb309f5aba6cb5a \ + --hash=sha256:8a28afb04baa55abf26df544e3e5c6534245d3daa5178bc4a8eeb48202060d0e \ + --hash=sha256:8b78d8a609a4b82c273257ee9d631ded7fac0d875bdcdccc109f3ee8328cfcb1 \ + --hash=sha256:8ce11cd4d62d11166f2b441e30ace226c19a3899a7cf0796f668fba49a9fb123 \ + --hash=sha256:8fff79bf5978c693c9b1a4d71e4a94fddfb5fe744eb062a318e15f4a2f63a550 \ + --hash=sha256:92263f7eca2f4af326cd20de8d16728d2602f7cfea02e790dcde9d83c365d7cc \ + --hash=sha256:93b3b2cc5cf1b8743660ce77a4f45f3f6d1172068207c1defc779a36eea6bb36 \ + --hash=sha256:95adae7b6c42a6c5b5b559b1a99149f090a57128155daeea91732c8d970d8644 \ + --hash=sha256:97ab7787092eb9b50fb47fa04f24c75b768a606af1bcba1957f07f128a7219e4 \ + --hash=sha256:9db5e3fcdcee89a78c04dffb3fe33c79f77bd741a624946db2591c81b2fc85b0 \ + --hash=sha256:a118e2e0b5ae6b0120d5efa5f866e58f2bb826067a646431da4d6a2bdae7950e \ + --hash=sha256:a2aecdb364b8a1802afdc7f9327d55dad5366bc97d8502d0f5854e50712dbc5f \ + --hash=sha256:a66aa5022bf81ab4b1bebfb009db4fd68e0c6d4307a1ce5ef6a26e5878dfc9e4 \ + --hash=sha256:a68766a3c58fde7f9aaa22b3786276f62ab2f594efb02d0a1421b6282e852e98 \ + --hash=sha256:aa2f963b4da26daf46231d9b9e0e2c9408a751f8f0d0f44d2de56d3caf51d294 \ + --hash=sha256:aa92ec1102eaff840ccd1021478af176a831f1bccb08e526ce844b7ddda85c22 \ + --hash=sha256:ac59c15e3f1465f722607800c68713f9fbc2f672b9eb649fe831da4019ae9b23 \ + --hash=sha256:ae8b03427410731469c4033934cf473426faff3e04b69d2dfb64a4281a3719f8 \ + --hash=sha256:afca7f78067dd27c2b848f1b234623d26b87529296c6c5652168cc1954f2f3b2 \ + --hash=sha256:b2d37d78297b39a9eb9eb92c0f6df98c706467282055419df141389b23f93362 \ + --hash=sha256:b3e71afc578b98512bfe7bdb822dd6bc57d4b0093b4b6e5487c1e96ad4ace242 \ + --hash=sha256:ba20bdf69bd127f66d0174d6f2a93e69045e0b4036dc1ca78e091bcc765830c4 \ + --hash=sha256:c108f8619e504140569ee7de3f97d234f0fbae338a7f9f360455071ef9855a95 \ + --hash=sha256:c23eb3263356d94858655b3e63f85ac5d50970c6e8febcdde7830209139cc37d \ + --hash=sha256:c5af897b45fa606b12464ccbe0014bbf8c09191e0a66aab6aa9d5cf6e77e0c94 \ + --hash=sha256:c7a80a9242963416bd81f99349d5f3fce1843c303bd404f204918b6d75a75fd6 \ + --hash=sha256:c7e84e0c0005e3bdc1a9211cd4e62c78ba80bc37b2365ef4410cd2007a9047f2 \ + --hash=sha256:cace89841c0599d736d3d74a27bc5821288bb47c5441923277afc6059d7fbcb4 \ + --hash=sha256:cd2d0f0ec9aa977a27731a3209ebbcacebebaf41f902bd453a928bfd281cf7f8 \ + --hash=sha256:d01de5e768328646e6a3fa9e562706f8f6641708c115c62588aef2b941a4f88e \ + --hash=sha256:d1028de43596a315e2720a9849ee79007ab742c06ad8b45a50db8cdb7ed4a82a \ + --hash=sha256:d27ce22ec453564770d29d03a9506d449efbb9fa13c00842262b2f6801c48cce \ + --hash=sha256:d29dd9c016f2078b43d0c357511e87eee5b05108f3dd603423cb389b89813969 \ + --hash=sha256:d31f0d1671e1534e395f9eb84a68e0fb670e1edb1fe819a9d7f564ae3bc4e53f \ + --hash=sha256:d4eb8ac7469b2a5d64b5b8c04f84d8bf3ad340f4514b98523805cbf46e3b3923 \ + --hash=sha256:d5e52d127045d6ae01a1e821acfad2f3a1866c54d0e837828538fabe8d9d1bd6 \ + --hash=sha256:d77f97e515688bd615c1d1f795d540f32542d514242067adcb8ef532504cb9ee \ + --hash=sha256:d8ed79b8f6372ca4254955005830fd61c1ccdd8c0fac6603e2c145c61dd95db6 \ + --hash=sha256:dc57a0baa3eeedd99fafaef7511b5a6ef4581494e8168ee086031744e2679467 \ + --hash=sha256:e09f671a54ce70b79a1fc1dc6da3072b7ef7251fadb894ed92d9aa8218465a5f \ + --hash=sha256:e22d1059b951e7ae7c20ef6b06afd10fb95e3c41bf3c4fbc874dba113321c193 \ + --hash=sha256:e37bd100d2c5d3ba35db9c7c5ba5a9228cbcffe5c4778dc824b164e5257813d7 \ + --hash=sha256:e51ae7d81c825761d941962450f50d041db028b7278e7b08930b4541b3e45cb9 \ + --hash=sha256:e545b51da9f9af5c67815ca0eb40676c0f016d0b0381c86f20451e35696c5f95 \ + --hash=sha256:e6302ca4ae283deb0af68d2fbf467474b8b6aedcd3dab4db187e07f94c109763 \ + --hash=sha256:e71bbb595973622b817c042bd943c3f3667e9c9983ce3d205f973f486fec98a7 \ + --hash=sha256:ec56a2266f32bc06ed3c3e2a8f58417ce02f7e0356edc89786e52db13c593c98 \ + --hash=sha256:ed1a9a204f317ef879b32f9af507d47e49cd5e7f8e8d5d96358c98373314fc60 \ + --hash=sha256:ed97c282ee4f994ef814042423a529df9497e3c666dca19be1d4cd1129dc7ade \ + --hash=sha256:ed98364e1c262cf5f9363c3eca8c2df37024f52a8fa1180a3610014f26eac51c \ + --hash=sha256:ee57b926940ba00bca7ba7041e665cc956e55ef482f851b9b65acb20d867e7a2 \ + --hash=sha256:f1d725b754e967e648046f00c4facc42d414840f5ccc670c5670f59f83693e4f \ + --hash=sha256:f8102ae93c0bc863b1d41ea0f4499c20a83229f52ed870850892df555187154a \ + --hash=sha256:fc1c64934b8faf7584924143eb9db4770bbdb16659626e1a1a4d9efbcb68d947 \ + --hash=sha256:ff95a9283de8a457e6b12989de3f9f5193430f375d64297d323a615ea52cbdb3 # via requests -claude-agent-sdk==0.1.48 +claude-agent-sdk==0.1.48 \ + --hash=sha256:0d37e60bd2b17efc3f927dccef080f14897ab62cd1d0d67a4abc8a0e2d4f1006 \ + --hash=sha256:39c1307daa17e42fa8a71180bb20af8a789d72d3891fc93519ff15540badcb83 \ + --hash=sha256:543d70acba468eccfff836965a14b8ac88cf90809aeeb88431dfcea3ee9a2fa9 \ + --hash=sha256:5761ff1d362e0f17c2b1bfd890d1c897f0aa81091e37bbd15b7d06f05ced552d \ + --hash=sha256:ee294d3f02936c0b826119ffbefcf88c67731cf8c2d2cb7111ccc97f76344272 # via -r requirements.txt -click==8.3.1 +click==8.3.1 \ + --hash=sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a \ + --hash=sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6 # via # flask # flask-socketio @@ -65,164 +415,1341 @@ click==8.3.1 # mlflow-skinny # typer # uvicorn -cloudpickle==3.1.2 +cloudpickle==3.1.2 \ + --hash=sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414 \ + --hash=sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a # via mlflow-skinny -contourpy==1.3.3 +contourpy==1.3.3 \ + --hash=sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69 \ + --hash=sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc \ + --hash=sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880 \ + --hash=sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a \ + --hash=sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8 \ + --hash=sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc \ + --hash=sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470 \ + --hash=sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5 \ + --hash=sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263 \ + --hash=sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b \ + --hash=sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5 \ + --hash=sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381 \ + --hash=sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3 \ + --hash=sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4 \ + --hash=sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e \ + --hash=sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f \ + --hash=sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772 \ + --hash=sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286 \ + --hash=sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42 \ + --hash=sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301 \ + --hash=sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77 \ + --hash=sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7 \ + --hash=sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411 \ + --hash=sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1 \ + --hash=sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9 \ + --hash=sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a \ + --hash=sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b \ + --hash=sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db \ + --hash=sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6 \ + --hash=sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620 \ + --hash=sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989 \ + --hash=sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea \ + --hash=sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67 \ + --hash=sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5 \ + --hash=sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d \ + --hash=sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36 \ + --hash=sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99 \ + --hash=sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1 \ + --hash=sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e \ + --hash=sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b \ + --hash=sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8 \ + --hash=sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d \ + --hash=sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7 \ + --hash=sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7 \ + --hash=sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339 \ + --hash=sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1 \ + --hash=sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659 \ + --hash=sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4 \ + --hash=sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f \ + --hash=sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20 \ + --hash=sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36 \ + --hash=sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb \ + --hash=sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d \ + --hash=sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8 \ + --hash=sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0 \ + --hash=sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b \ + --hash=sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7 \ + --hash=sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe \ + --hash=sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77 \ + --hash=sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497 \ + --hash=sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd \ + --hash=sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1 \ + --hash=sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216 \ + --hash=sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13 \ + --hash=sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae \ + --hash=sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae \ + --hash=sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77 \ + --hash=sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3 \ + --hash=sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f \ + --hash=sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff \ + --hash=sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9 \ + --hash=sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a # via matplotlib -cryptography==46.0.5 +cryptography==46.0.5 \ + --hash=sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72 \ + --hash=sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235 \ + --hash=sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9 \ + --hash=sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356 \ + --hash=sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257 \ + --hash=sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad \ + --hash=sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4 \ + --hash=sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c \ + --hash=sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614 \ + --hash=sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed \ + --hash=sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31 \ + --hash=sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229 \ + --hash=sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0 \ + --hash=sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731 \ + --hash=sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b \ + --hash=sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4 \ + --hash=sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4 \ + --hash=sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263 \ + --hash=sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595 \ + --hash=sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1 \ + --hash=sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678 \ + --hash=sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48 \ + --hash=sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76 \ + --hash=sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0 \ + --hash=sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18 \ + --hash=sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d \ + --hash=sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d \ + --hash=sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1 \ + --hash=sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981 \ + --hash=sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7 \ + --hash=sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82 \ + --hash=sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2 \ + --hash=sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4 \ + --hash=sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663 \ + --hash=sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c \ + --hash=sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d \ + --hash=sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a \ + --hash=sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a \ + --hash=sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d \ + --hash=sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b \ + --hash=sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a \ + --hash=sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826 \ + --hash=sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee \ + --hash=sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9 \ + --hash=sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648 \ + --hash=sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da \ + --hash=sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2 \ + --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2 \ + --hash=sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87 # via # google-auth # mlflow # pyjwt -cycler==0.12.1 +cycler==0.12.1 \ + --hash=sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30 \ + --hash=sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c # via matplotlib -databricks-sdk==0.99.0 +databricks-sdk==0.99.0 \ + --hash=sha256:13ae35b064277074a79fcd5265260e92b352b5eb4e950438dcee895951bd86fd \ + --hash=sha256:c0a6740bf21d430daa85461e193197ce1d0ac12209ce20e3cb9b67b32fe2bcfd # via # -r requirements.txt # mlflow-skinny # mlflow-tracing -deprecated==1.3.1 +deprecated==1.3.1 \ + --hash=sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f \ + --hash=sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223 # via limits -distro==1.9.0 +distro==1.9.0 \ + --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 # via openai -docker==7.1.0 +docker==7.1.0 \ + --hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \ + --hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0 # via mlflow -fastapi==0.135.1 +fastapi==0.135.1 \ + --hash=sha256:46e2fc5745924b7c840f71ddd277382af29ce1cdb7d5eab5bf697e3fb9999c9e \ + --hash=sha256:d04115b508d936d254cea545b7312ecaa58a7b3a0f84952535b4c9afae7668cd # via # mlflow # mlflow-skinny -fastuuid==0.14.0 +fastuuid==0.14.0 \ + --hash=sha256:05a8dde1f395e0c9b4be515b7a521403d1e8349443e7641761af07c7ad1624b1 \ + --hash=sha256:0737606764b29785566f968bd8005eace73d3666bd0862f33a760796e26d1ede \ + --hash=sha256:089c18018fdbdda88a6dafd7d139f8703a1e7c799618e33ea25eb52503d28a11 \ + --hash=sha256:09098762aad4f8da3a888eb9ae01c84430c907a297b97166b8abc07b640f2995 \ + --hash=sha256:09378a05020e3e4883dfdab438926f31fea15fd17604908f3d39cbeb22a0b4dc \ + --hash=sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796 \ + --hash=sha256:0df14e92e7ad3276327631c9e7cec09e32572ce82089c55cb1bb8df71cf394ed \ + --hash=sha256:12ac85024637586a5b69645e7ed986f7535106ed3013640a393a03e461740cb7 \ + --hash=sha256:1383fff584fa249b16329a059c68ad45d030d5a4b70fb7c73a08d98fd53bcdab \ + --hash=sha256:139d7ff12bb400b4a0c76be64c28cbe2e2edf60b09826cbfd85f33ed3d0bbe8b \ + --hash=sha256:13ec4f2c3b04271f62be2e1ce7e95ad2dd1cf97e94503a3760db739afbd48f00 \ + --hash=sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26 \ + --hash=sha256:193ca10ff553cf3cc461572da83b5780fc0e3eea28659c16f89ae5202f3958d4 \ + --hash=sha256:1a771f135ab4523eb786e95493803942a5d1fc1610915f131b363f55af53b219 \ + --hash=sha256:1bf539a7a95f35b419f9ad105d5a8a35036df35fdafae48fb2fd2e5f318f0d75 \ + --hash=sha256:1ca61b592120cf314cfd66e662a5b54a578c5a15b26305e1b8b618a6f22df714 \ + --hash=sha256:1e3cc56742f76cd25ecb98e4b82a25f978ccffba02e4bdce8aba857b6d85d87b \ + --hash=sha256:1e690d48f923c253f28151b3a6b4e335f2b06bf669c68a02665bc150b7839e94 \ + --hash=sha256:2b29e23c97e77c3a9514d70ce343571e469098ac7f5a269320a0f0b3e193ab36 \ + --hash=sha256:2dce5d0756f046fa792a40763f36accd7e466525c5710d2195a038f93ff96346 \ + --hash=sha256:2ec3d94e13712a133137b2805073b65ecef4a47217d5bac15d8ac62376cefdb4 \ + --hash=sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8 \ + --hash=sha256:2fc37479517d4d70c08696960fad85494a8a7a0af4e93e9a00af04d74c59f9e3 \ + --hash=sha256:33e678459cf4addaedd9936bbb038e35b3f6b2061330fd8f2f6a1d80414c0f87 \ + --hash=sha256:3964bab460c528692c70ab6b2e469dd7a7b152fbe8c18616c58d34c93a6cf8d4 \ + --hash=sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8 \ + --hash=sha256:448aa6833f7a84bfe37dd47e33df83250f404d591eb83527fa2cac8d1e57d7f3 \ + --hash=sha256:47c821f2dfe95909ead0085d4cb18d5149bca704a2b03e03fb3f81a5202d8cea \ + --hash=sha256:4edc56b877d960b4eda2c4232f953a61490c3134da94f3c28af129fb9c62a4f6 \ + --hash=sha256:5816d41f81782b209843e52fdef757a361b448d782452d96abedc53d545da722 \ + --hash=sha256:6e6243d40f6c793c3e2ee14c13769e341b90be5ef0c23c82fa6515a96145181a \ + --hash=sha256:6fbc49a86173e7f074b1a9ec8cf12ca0d54d8070a85a06ebf0e76c309b84f0d0 \ + --hash=sha256:73657c9f778aba530bc96a943d30e1a7c80edb8278df77894fe9457540df4f85 \ + --hash=sha256:73946cb950c8caf65127d4e9a325e2b6be0442a224fd51ba3b6ac44e1912ce34 \ + --hash=sha256:77a09cb7427e7af74c594e409f7731a0cf887221de2f698e1ca0ebf0f3139021 \ + --hash=sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a \ + --hash=sha256:7a3c0bca61eacc1843ea97b288d6789fbad7400d16db24e36a66c28c268cfe3d \ + --hash=sha256:7f2f3efade4937fae4e77efae1af571902263de7b78a0aee1a1653795a093b2a \ + --hash=sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09 \ + --hash=sha256:83cffc144dc93eb604b87b179837f2ce2af44871a7b323f2bfed40e8acb40ba8 \ + --hash=sha256:84b0779c5abbdec2a9511d5ffbfcd2e53079bf889824b32be170c0d8ef5fc74c \ + --hash=sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176 \ + --hash=sha256:9a133bf9cc78fdbd1179cb58a59ad0100aa32d8675508150f3658814aeefeaa4 \ + --hash=sha256:9bd57289daf7b153bfa3e8013446aa144ce5e8c825e9e366d455155ede5ea2dc \ + --hash=sha256:a0809f8cc5731c066c909047f9a314d5f536c871a7a22e815cc4967c110ac9ad \ + --hash=sha256:a6f46790d59ab38c6aa0e35c681c0484b50dc0acf9e2679c005d61e019313c24 \ + --hash=sha256:a8a0dfea3972200f72d4c7df02c8ac70bad1bb4c58d7e0ec1e6f341679073a7f \ + --hash=sha256:aa75b6657ec129d0abded3bec745e6f7ab642e6dba3a5272a68247e85f5f316f \ + --hash=sha256:ab32f74bd56565b186f036e33129da77db8be09178cd2f5206a5d4035fb2a23f \ + --hash=sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741 \ + --hash=sha256:ac60fc860cdf3c3f327374db87ab8e064c86566ca8c49d2e30df15eda1b0c2d5 \ + --hash=sha256:ae64ba730d179f439b0736208b4c279b8bc9c089b102aec23f86512ea458c8a4 \ + --hash=sha256:af5967c666b7d6a377098849b07f83462c4fedbafcf8eb8bc8ff05dcbe8aa209 \ + --hash=sha256:b2fdd48b5e4236df145a149d7125badb28e0a383372add3fbaac9a6b7a394470 \ + --hash=sha256:b852a870a61cfc26c884af205d502881a2e59cc07076b60ab4a951cc0c94d1ad \ + --hash=sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057 \ + --hash=sha256:bbb0c4b15d66b435d2538f3827f05e44e2baafcc003dd7d8472dc67807ab8fd8 \ + --hash=sha256:bcc96ee819c282e7c09b2eed2b9bd13084e3b749fdb2faf58c318d498df2efbe \ + --hash=sha256:c0a94245afae4d7af8c43b3159d5e3934c53f47140be0be624b96acd672ceb73 \ + --hash=sha256:c0eb25f0fd935e376ac4334927a59e7c823b36062080e2e13acbaf2af15db836 \ + --hash=sha256:c3091e63acf42f56a6f74dc65cfdb6f99bfc79b5913c8a9ac498eb7ca09770a8 \ + --hash=sha256:c501561e025b7aea3508719c5801c360c711d5218fc4ad5d77bf1c37c1a75779 \ + --hash=sha256:c7502d6f54cd08024c3ea9b3514e2d6f190feb2f46e6dbcd3747882264bb5f7b \ + --hash=sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d \ + --hash=sha256:cb9a030f609194b679e1660f7e32733b7a0f332d519c5d5a6a0a580991290022 \ + --hash=sha256:cd5a7f648d4365b41dbf0e38fe8da4884e57bed4e77c83598e076ac0c93995e7 \ + --hash=sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070 \ + --hash=sha256:d31f8c257046b5617fc6af9c69be066d2412bdef1edaa4bdf6a214cf57806105 \ + --hash=sha256:d55b7e96531216fc4f071909e33e35e5bfa47962ae67d9e84b00a04d6e8b7173 \ + --hash=sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397 \ + --hash=sha256:de01280eabcd82f7542828ecd67ebf1551d37203ecdfd7ab1f2e534edb78d505 \ + --hash=sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a \ + --hash=sha256:e0976c0dff7e222513d206e06341503f07423aceb1db0b83ff6851c008ceee06 \ + --hash=sha256:e150eab56c95dc9e3fefc234a0eedb342fac433dacc273cd4d150a5b0871e1fa \ + --hash=sha256:e23fc6a83f112de4be0cc1990e5b127c27663ae43f866353166f87df58e73d06 \ + --hash=sha256:ec27778c6ca3393ef662e2762dba8af13f4ec1aaa32d08d77f71f2a70ae9feb8 \ + --hash=sha256:f54d5b36c56a2d5e1a31e73b950b28a0d83eb0c37b91d10408875a5a29494bad \ + --hash=sha256:f74631b8322d2780ebcf2d2d75d58045c3e9378625ec51865fe0b5620800c39d # via litellm -filelock==3.25.2 +filelock==3.25.2 \ + --hash=sha256:b64ece2b38f4ca29dd3e810287aa8c48182bbecd1ae6e9ae126c9b35f1382694 \ + --hash=sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70 # via huggingface-hub -flask==3.1.3 +flask==3.1.3 \ + --hash=sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb \ + --hash=sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c # via # -r requirements.txt # flask-cors # flask-socketio # mlflow -flask-cors==6.0.2 +flask-cors==6.0.2 \ + --hash=sha256:6e118f3698249ae33e429760db98ce032a8bf9913638d085ca0f4c5534ad2423 \ + --hash=sha256:e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a # via mlflow -flask-socketio==5.6.1 +flask-socketio==5.6.1 \ + --hash=sha256:51a3f71b28b4476c650829607e3a993e076034db6c3cc31f718f0a4b45939d42 \ + --hash=sha256:fe5bd995c3ed4da9a98f335d0d830fa1a19d84a64789f6265642a671fdacaeac # via -r requirements.txt -fonttools==4.62.0 +fonttools==4.62.0 \ + --hash=sha256:0361a7d41d86937f1f752717c19f719d0fde064d3011038f9f19bdf5fc2f5c95 \ + --hash=sha256:03c6068adfdc67c565d217e92386b1cdd951abd4240d65180cec62fa74ba31b2 \ + --hash=sha256:090e74ac86e68c20150e665ef8e7e0c20cb9f8b395302c9419fa2e4d332c3b51 \ + --hash=sha256:0dc477c12b8076b4eb9af2e440421b0433ffa9e1dcb39e0640a6c94665ed1098 \ + --hash=sha256:106aec9226f9498fc5345125ff7200842c01eda273ae038f5049b0916907acee \ + --hash=sha256:13b663fb197334de84db790353d59da2a7288fd14e9be329f5debc63ec0500a5 \ + --hash=sha256:153afc3012ff8761b1733e8fbe5d98623409774c44ffd88fbcb780e240c11d13 \ + --hash=sha256:15d86b96c79013320f13bc1b15f94789edb376c0a2d22fb6088f33637e8dfcbc \ + --hash=sha256:196cafef9aeec5258425bd31a4e9a414b2ee0d1557bca184d7923d3d3bcd90f9 \ + --hash=sha256:22bde4dc12a9e09b5ced77f3b5053d96cf10c4976c6ac0dee293418ef289d221 \ + --hash=sha256:273acb61f316d07570a80ed5ff0a14a23700eedbec0ad968b949abaa4d3f6bb5 \ + --hash=sha256:274c8b8a87e439faf565d3bcd3f9f9e31bca7740755776a4a90a4bfeaa722efa \ + --hash=sha256:28a9ea2a7467a816d1bec22658b0cce4443ac60abac3e293bdee78beb74588f3 \ + --hash=sha256:31a804c16d76038cc4e3826e07678efb0a02dc4f15396ea8e07088adbfb2578e \ + --hash=sha256:37a73e5e38fd05c637daede6ffed5f3496096be7df6e4a3198d32af038f87527 \ + --hash=sha256:3e2ff573de2775508c8a366351fb901c4ced5dc6cf2d87dd15c973bedcdd5216 \ + --hash=sha256:3f9e20c4618f1e04190c802acae6dc337cb6db9fa61e492fd97cd5c5a9ff6d07 \ + --hash=sha256:42c7848fa8836ab92c23b1617c407a905642521ff2d7897fe2bf8381530172f1 \ + --hash=sha256:44956b003151d5a289eba6c71fe590d63509267c37e26de1766ba15d9c589582 \ + --hash=sha256:4da779e8f342a32856075ddb193b2a024ad900bc04ecb744014c32409ae871ed \ + --hash=sha256:4f16c07e5250d5d71d0f990a59460bc5620c3cc456121f2cfb5b60475699905f \ + --hash=sha256:4fa5a9c716e2f75ef34b5a5c2ca0ee4848d795daa7e6792bf30fd4abf8993449 \ + --hash=sha256:55b189a1b3033860a38e4e5bd0626c5aa25c7ce9caee7bc784a8caec7a675401 \ + --hash=sha256:579f35c121528a50c96bf6fcb6a393e81e7f896d4326bf40e379f1c971603db9 \ + --hash=sha256:591220d5333264b1df0d3285adbdfe2af4f6a45bbf9ca2b485f97c9f577c49ff \ + --hash=sha256:5ae611294f768d413949fd12693a8cba0e6332fbc1e07aba60121be35eac68d0 \ + --hash=sha256:6247e58b96b982709cd569a91a2ba935d406dccf17b6aa615afaed37ac3856aa \ + --hash=sha256:625f5cbeb0b8f4e42343eaeb4bc2786718ddd84760a2f5e55fdd3db049047c00 \ + --hash=sha256:62b6a3d0028e458e9b59501cf7124a84cd69681c433570e4861aff4fb54a236c \ + --hash=sha256:658ab837c878c4d2a652fcbb319547ea41693890e6434cf619e66f79387af3b8 \ + --hash=sha256:6826a5aa53fb6def8a66bf423939745f415546c4e92478a7c531b8b6282b6c3b \ + --hash=sha256:7199c73b326bad892f1cb53ffdd002128bfd58a89b8f662204fbf1daf8d62e85 \ + --hash=sha256:75064f19a10c50c74b336aa5ebe7b1f89fd0fb5255807bfd4b0c6317098f4af3 \ + --hash=sha256:825f98cd14907c74a4d0a3f7db8570886ffce9c6369fed1385020febf919abf6 \ + --hash=sha256:83c6524c5b93bad9c2939d88e619fedc62e913c19e673f25d5ab74e7a5d074e5 \ + --hash=sha256:840632ea9c1eab7b7f01c369e408c0721c287dfd7500ab937398430689852fd1 \ + --hash=sha256:8f086120e8be9e99ca1288aa5ce519833f93fe0ec6ebad2380c1dee18781f0b5 \ + --hash=sha256:93e27131a5a0ae82aaadcffe309b1bae195f6711689722af026862bede05c07c \ + --hash=sha256:966557078b55e697f65300b18025c54e872d7908d1899b7314d7c16e64868cb2 \ + --hash=sha256:9bf75eb69330e34ad2a096fac67887102c8537991eb6cac1507fc835bbb70e0a \ + --hash=sha256:9cf34861145b516cddd19b07ae6f4a61ea1c6326031b960ec9ddce8ee815e888 \ + --hash=sha256:a5f974006d14f735c6c878fc4b117ad031dc93638ddcc450ca69f8fd64d5e104 \ + --hash=sha256:b448075f32708e8fb377fe7687f769a5f51a027172c591ba9a58693631b077a8 \ + --hash=sha256:c858030560f92a054444c6e46745227bfd3bb4e55383c80d79462cd47289e4b5 \ + --hash=sha256:d28d5baacb0017d384df14722a63abe6e0230d8ce642b1615a27d78ffe3bc983 \ + --hash=sha256:d31558890f3fa00d4f937d12708f90c7c142c803c23eaeb395a71f987a77ebe3 \ + --hash=sha256:d4108c12773b3c97aa592311557c405d5b4fc03db2b969ed928fcf68e7b3c887 \ + --hash=sha256:d732938633681d6e2324e601b79e93f7f72395ec8681f9cdae5a8c08bc167e72 \ + --hash=sha256:e5f1fa8cc9f1a56a3e33ee6b954d6d9235e6b9d11eb7a6c9dfe2c2f829dc24db \ + --hash=sha256:f8c8ea812f82db1e884b9cdb663080453e28f0f9a1f5027a5adb59c4cc8d38d1 # via matplotlib -frozenlist==1.8.0 +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd # via # aiohttp # aiosignal -fsspec==2026.2.0 +fsspec==2026.2.0 \ + --hash=sha256:6544e34b16869f5aacd5b90bdf1a71acb37792ea3ddf6125ee69a22a53fb8bff \ + --hash=sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437 # via huggingface-hub -gepa==0.1.0 +gepa==0.1.0 \ + --hash=sha256:4e3f8fe8ca20169e60518b2e9d416e8c4a579459848adffdcad12223fbf9643e \ + --hash=sha256:f8b3d7918d4cdcf8593f39ef1cc757c4ba1a4e6793e3ffb622e6c0bc60a1efd9 # via mlflow -gitdb==4.0.12 +gitdb==4.0.12 \ + --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ + --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf # via gitpython -gitpython==3.1.46 +gitpython==3.1.46 \ + --hash=sha256:400124c7d0ef4ea03f7310ac2fbf7151e09ff97f2a3288d64a440c584a29c37f \ + --hash=sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058 # via mlflow-skinny -google-auth==2.49.0 +google-auth==2.49.0 \ + --hash=sha256:9cc2d9259d3700d7a257681f81052db6737495a1a46b610597f4b8bafe5286ae \ + --hash=sha256:f893ef7307f19cf53700b7e2f61b5a6affe3aa0edf9943b13788920ab92d8d87 # via databricks-sdk -googleapis-common-protos==1.73.0 +googleapis-common-protos==1.73.0 \ + --hash=sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a \ + --hash=sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8 # via opentelemetry-exporter-otlp-proto-grpc -graphene==3.4.3 +graphene==3.4.3 \ + --hash=sha256:2a3786948ce75fe7e078443d37f609cbe5bb36ad8d6b828740ad3b95ed1a0aaa \ + --hash=sha256:820db6289754c181007a150db1f7fff544b94142b556d12e3ebc777a7bf36c71 # via mlflow -graphql-core==3.2.8 +graphql-core==3.2.8 \ + --hash=sha256:015457da5d996c924ddf57a43f4e959b0b94fb695b85ed4c29446e508ed65cf3 \ + --hash=sha256:cbee07bee1b3ed5e531723685369039f32ff815ef60166686e0162f540f1520c # via # graphene # graphql-relay -graphql-relay==3.2.0 +graphql-relay==3.2.0 \ + --hash=sha256:1ff1c51298356e481a0be009ccdff249832ce53f30559c1338f22a0e0d17250c \ + --hash=sha256:c9b22bd28b170ba1fe674c74384a8ff30a76c8e26f88ac3aa1584dd3179953e5 # via graphene -grpcio==1.78.0 +grpcio==1.78.0 \ + --hash=sha256:082653eecbdf290e6e3e2c276ab2c54b9e7c299e07f4221872380312d8cf395e \ + --hash=sha256:10a9a644b5dd5aec3b82b5b0b90d41c0fa94c85ef42cb42cf78a23291ddb5e7d \ + --hash=sha256:12a771591ae40bc65ba67048fa52ef4f0e6db8279e595fd349f9dfddeef571f9 \ + --hash=sha256:185dea0d5260cbb2d224c507bf2a5444d5abbb1fa3594c1ed7e4c709d5eb8383 \ + --hash=sha256:1afa62af6e23f88629f2b29ec9e52ec7c65a7176c1e0a83292b93c76ca882558 \ + --hash=sha256:2045397e63a7a0ee7957c25f7dbb36ddc110e0cfb418403d110c0a7a68a844e9 \ + --hash=sha256:207db540302c884b8848036b80db352a832b99dfdf41db1eb554c2c2c7800f65 \ + --hash=sha256:271c73e6e5676afe4fc52907686670c7cea22ab2310b76a59b678403ed40d670 \ + --hash=sha256:2777b783f6c13b92bd7b716667452c329eefd646bfb3f2e9dabea2e05dbd34f6 \ + --hash=sha256:2bf5e2e163b356978b23652c4818ce4759d40f4712ee9ec5a83c4be6f8c23a3a \ + --hash=sha256:35eb275bf1751d2ffbd8f57cdbc46058e857cf3971041521b78b7db94bdaf127 \ + --hash=sha256:391e93548644e6b2726f1bb84ed60048d4bcc424ce5e4af0843d28ca0b754fec \ + --hash=sha256:3c586ac70e855c721bda8f548d38c3ca66ac791dc49b66a8281a1f99db85e452 \ + --hash=sha256:3f8904a8165ab21e07e58bf3e30a73f4dffc7a1e0dbc32d51c61b5360d26f43e \ + --hash=sha256:459ab414b35f4496138d0ecd735fed26f1318af5e52cb1efbc82a09f0d5aa911 \ + --hash=sha256:4c5533d03a6cbd7f56acfc9cfb44ea64f63d29091e40e44010d34178d392d7eb \ + --hash=sha256:51b13f9aed9d59ee389ad666b8c2214cc87b5de258fa712f9ab05f922e3896c6 \ + --hash=sha256:5361a0630a7fdb58a6a97638ab70e1dae2893c4d08d7aba64ded28bb9e7a29df \ + --hash=sha256:5397fff416b79e4b284959642a4e95ac4b0f1ece82c9993658e0e477d40551ec \ + --hash=sha256:57bab6deef2f4f1ca76cc04565df38dc5713ae6c17de690721bdf30cb1e0545c \ + --hash=sha256:6092beabe1966a3229f599d7088b38dfc8ffa1608b5b5cdda31e591e6500f856 \ + --hash=sha256:684083fd383e9dc04c794adb838d4faea08b291ce81f64ecd08e4577c7398adf \ + --hash=sha256:735e38e176a88ce41840c21bb49098ab66177c64c82426e24e0082500cc68af5 \ + --hash=sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5 \ + --hash=sha256:748b6138585379c737adc08aeffd21222abbda1a86a0dca2a39682feb9196c20 \ + --hash=sha256:74be1268d1439eaaf552c698cdb11cd594f0c49295ae6bb72c34ee31abbe611b \ + --hash=sha256:7cc47943d524ee0096f973e1081cb8f4f17a4615f2116882a5f1416e4cfe92b5 \ + --hash=sha256:859b13906ce098c0b493af92142ad051bf64c7870fa58a123911c88606714996 \ + --hash=sha256:85f93781028ec63f383f6bc90db785a016319c561cc11151fbb7b34e0d012303 \ + --hash=sha256:86ce2371bfd7f212cf60d8517e5e854475c2c43ce14aa910e136ace72c6db6c1 \ + --hash=sha256:86f85dd7c947baa707078a236288a289044836d4b640962018ceb9cd1f899af5 \ + --hash=sha256:8dfffba826efcf366b1e3ccc37e67afe676f290e13a3b48d31a46739f80a8724 \ + --hash=sha256:8f2ac84905d12918e4e55a16da17939eb63e433dc11b677267c35568aa63fc84 \ + --hash=sha256:94309f498bcc07e5a7d16089ab984d42ad96af1d94b5a4eb966a266d9fcabf68 \ + --hash=sha256:94f95cf5d532d0e717eed4fc1810e8e6eded04621342ec54c89a7c2f14b581bf \ + --hash=sha256:9566fe4ababbb2610c39190791e5b829869351d14369603702e890ef3ad2d06e \ + --hash=sha256:9dca934f24c732750389ce49d638069c3892ad065df86cb465b3fa3012b70c9e \ + --hash=sha256:a9f136fbafe7ccf4ac7e8e0c28b31066e810be52d6e344ef954a3a70234e1702 \ + --hash=sha256:ab399ef5e3cd2a721b1038a0f3021001f19c5ab279f145e1146bb0b9f1b2b12c \ + --hash=sha256:b0c689c02947d636bc7fab3e30cc3a3445cca99c834dfb77cd4a6cabfc1c5597 \ + --hash=sha256:b2342d87af32790f934a79c3112641e7b27d63c261b8b4395350dad43eff1dc7 \ + --hash=sha256:b58f37edab4a3881bc6c9bca52670610e0c9ca14e2ea3cf9debf185b870457fb \ + --hash=sha256:bd8cb8026e5f5b50498a3c4f196f57f9db344dad829ffae16b82e4fdbaea2813 \ + --hash=sha256:be63c88b32e6c0f1429f1398ca5c09bc64b0d80950c8bb7807d7d7fb36fb84c7 \ + --hash=sha256:c3f293fdc675ccba4db5a561048cca627b5e7bd1c8a6973ffedabe7d116e22e2 \ + --hash=sha256:c41bc64626db62e72afec66b0c8a0da76491510015417c127bfc53b2fe6d7f7f \ + --hash=sha256:ce3a90455492bf8bfa38e56fbbe1dbd4f872a3d8eeaf7337dc3b1c8aa28c271b \ + --hash=sha256:ce7599575eeb25c0f4dc1be59cada6219f3b56176f799627f44088b21381a28a \ + --hash=sha256:dce09d6116df20a96acfdbf85e4866258c3758180e8c49845d6ba8248b6d0bbb \ + --hash=sha256:de8cb00d1483a412a06394b8303feec5dcb3b55f81d83aa216dbb6a0b86a94f5 \ + --hash=sha256:df2c8f3141f7cbd112a6ebbd760290b5849cda01884554f7c67acc14e7b1758a \ + --hash=sha256:e87cbc002b6f440482b3519e36e1313eb5443e9e9e73d6a52d43bd2004fcfd8e \ + --hash=sha256:e888474dee2f59ff68130f8a397792d8cb8e17e6b3434339657ba4ee90845a8c \ + --hash=sha256:f12857d24d98441af6a1d5c87442d624411db486f7ba12550b07788f74b67b04 \ + --hash=sha256:f2d4e43ee362adfc05994ed479334d5a451ab7bc3f3fee1b796b8ca66895acb4 \ + --hash=sha256:f3d6379493e18ad4d39537a82371c5281e153e963cecb13f953ebac155756525 \ + --hash=sha256:f8dff3d9777e5d2703a962ee5c286c239bf0ba173877cc68dc02c17d042e29de \ + --hash=sha256:f9ab915a267fc47c7e88c387a3a28325b58c898e23d4995f765728f4e3dedb97 \ + --hash=sha256:fbe6e89c7ffb48518384068321621b2a69cab509f58e40e4399fdd378fa6d074 \ + --hash=sha256:fd5f135b1bd58ab088930b3c613455796dfa0393626a6972663ccdda5b4ac6ce \ + --hash=sha256:ff870aebe9a93a85283837801d35cd5f8814fe2ad01e606861a7fb47c762a2b7 # via opentelemetry-exporter-otlp-proto-grpc -gunicorn==25.1.0 +gunicorn==25.1.0 \ + --hash=sha256:1426611d959fa77e7de89f8c0f32eed6aa03ee735f98c01efba3e281b1c47616 \ + --hash=sha256:d0b1236ccf27f72cfe14bce7caadf467186f19e865094ca84221424e839b8b8b # via mlflow -h11==0.16.0 +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via # httpcore # uvicorn # wsproto -hf-xet==1.4.0 +hf-xet==1.4.0 \ + --hash=sha256:01de78b1ceddf8b38da001f7cc728b3bc3eb956948b18e8a1997ad6fc80fbe9d \ + --hash=sha256:06da3797f1fdd9a8f8dbc8c1bddfa0b914789b14580c375d29c32ee35c2c66ca \ + --hash=sha256:07ffdbf7568fa3245b24d949f0f3790b5276fb7293a5554ac4ec02e5f7e2b38d \ + --hash=sha256:1818c2e5d6f15354c595d5111c6eb0e5a30a6c5c1a43eeaec20f19607cff0b34 \ + --hash=sha256:30b9d8f384ccec848124d51d883e91f3c88d430589e02a7b6d867730ab8d53ac \ + --hash=sha256:3a5d9cb25095ceb3beab4843ae2d1b3e5746371ddbf2e5849f7be6a7d6f44df4 \ + --hash=sha256:48e6ba7422b0885c9bbd8ac8fdf5c4e1306c3499b82d489944609cc4eae8ecbd \ + --hash=sha256:5d0c38d2a280d814280b8c15eead4a43c9781e7bf6fc37843cffab06dcdc76b9 \ + --hash=sha256:6a883f0250682ea888a1bd0af0631feda377e59ad7aae6fb75860ecee7ae0f93 \ + --hash=sha256:6cac8616e7a974105c3494735313f5ab0fb79b5accadec1a7a992859a15536a9 \ + --hash=sha256:70764d295f485db9cc9a6af76634ea00ec4f96311be7485f8f2b6144739b4ccf \ + --hash=sha256:76725fcbc5f59b23ac778f097d3029d6623e3cf6f4057d99d1fce1a7e3cff8fc \ + --hash=sha256:76f1f73bee81a6e6f608b583908aa24c50004965358ac92c1dc01080a21bcd09 \ + --hash=sha256:8d6d7816d01e0fa33f315c8ca21b05eca0ce4cdc314f13b81d953e46cc6db11d \ + --hash=sha256:981d2b5222c3baadf9567c135cf1d1073786f546b7745686978d46b5df179e16 \ + --hash=sha256:99e1d9255fe8ecdf57149bb0543d49e7b7bd8d491ddf431eb57e114253274df5 \ + --hash=sha256:9b777674499dc037317db372c90a2dd91329b5f1ee93c645bb89155bb974f5bf \ + --hash=sha256:9c0c9f052738a024073d332c573275c8e33697a3ef3f5dd2fb4ef98216e1e74a \ + --hash=sha256:9d3bd2a1e289f772c715ca88cdca8ceb3d8b5c9186534d5925410e531d849a3e \ + --hash=sha256:b25f06ce42bd2d5f2e79d4a2d72f783d3ac91827c80d34a38cf8e5290dd717b0 \ + --hash=sha256:b6f3729335fbc4baef60fe14fe32ef13ac9d377bdc898148c541e20c6056b504 \ + --hash=sha256:cb8d9549122b5b42f34b23b14c6b662a88a586a919d418c774d8dbbc4b3ce2aa \ + --hash=sha256:cc8bd050349d0d7995ce7b3a3a18732a2a8062ce118a82431602088abb373428 \ + --hash=sha256:e2731044f3a18442f9f7a3dcf03b96af13dee311f03846a1df1f0553a3ea0fc6 \ + --hash=sha256:f44b2324be75bfa399735996ac299fd478684c48ce47d12a42b5f24b1a99ccb8 # via huggingface-hub -httpcore==1.0.9 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 # via httpx -httptools==0.7.1 +httptools==0.7.1 \ + --hash=sha256:04c6c0e6c5fb0739c5b8a9eb046d298650a0ff38cf42537fc372b28dc7e4472c \ + --hash=sha256:0d92b10dbf0b3da4823cde6a96d18e6ae358a9daa741c71448975f6a2c339cad \ + --hash=sha256:0e68b8582f4ea9166be62926077a3334064d422cf08ab87d8b74664f8e9058e1 \ + --hash=sha256:11d01b0ff1fe02c4c32d60af61a4d613b74fad069e47e06e9067758c01e9ac78 \ + --hash=sha256:135fbe974b3718eada677229312e97f3b31f8a9c8ffa3ae6f565bf808d5b6bcb \ + --hash=sha256:2c15f37ef679ab9ecc06bfc4e6e8628c32a8e4b305459de7cf6785acd57e4d03 \ + --hash=sha256:322d00c2068d125bd570f7bf78b2d367dad02b919d8581d7476d8b75b294e3e6 \ + --hash=sha256:379b479408b8747f47f3b253326183d7c009a3936518cdb70db58cffd369d9df \ + --hash=sha256:38e0c83a2ea9746ebbd643bdfb521b9aa4a91703e2cd705c20443405d2fd16a5 \ + --hash=sha256:3e14f530fefa7499334a79b0cf7e7cd2992870eb893526fb097d51b4f2d0f321 \ + --hash=sha256:44c8f4347d4b31269c8a9205d8a5ee2df5322b09bbbd30f8f862185bb6b05346 \ + --hash=sha256:465275d76db4d554918aba40bf1cbebe324670f3dfc979eaffaa5d108e2ed650 \ + --hash=sha256:474d3b7ab469fefcca3697a10d11a32ee2b9573250206ba1e50d5980910da657 \ + --hash=sha256:49794f9250188a57fa73c706b46cb21a313edb00d337ca4ce1a011fe3c760b28 \ + --hash=sha256:5ddbd045cfcb073db2449563dd479057f2c2b681ebc232380e63ef15edc9c023 \ + --hash=sha256:601b7628de7504077dd3dcb3791c6b8694bbd967148a6d1f01806509254fb1ca \ + --hash=sha256:654968cb6b6c77e37b832a9be3d3ecabb243bbe7a0b8f65fbc5b6b04c8fcabed \ + --hash=sha256:69d4f9705c405ae3ee83d6a12283dc9feba8cc6aaec671b412917e644ab4fa66 \ + --hash=sha256:6babce6cfa2a99545c60bfef8bee0cc0545413cb0018f617c8059a30ad985de3 \ + --hash=sha256:7347714368fb2b335e9063bc2b96f2f87a9ceffcd9758ac295f8bbcd3ffbc0ca \ + --hash=sha256:7aea2e3c3953521c3c51106ee11487a910d45586e351202474d45472db7d72d3 \ + --hash=sha256:7fe6e96090df46b36ccfaf746f03034e5ab723162bc51b0a4cf58305324036f2 \ + --hash=sha256:84d86c1e5afdc479a6fdabf570be0d3eb791df0ae727e8dbc0259ed1249998d4 \ + --hash=sha256:a3c3b7366bb6c7b96bd72d0dbe7f7d5eead261361f013be5f6d9590465ea1c70 \ + --hash=sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9 \ + --hash=sha256:ac50afa68945df63ec7a2707c506bd02239272288add34539a2ef527254626a4 \ + --hash=sha256:aeefa0648362bb97a7d6b5ff770bfb774930a327d7f65f8208394856862de517 \ + --hash=sha256:b580968316348b474b020edf3988eecd5d6eec4634ee6561e72ae3a2a0e00a8a \ + --hash=sha256:c08fe65728b8d70b6923ce31e3956f859d5e1e8548e6f22ec520a962c6757270 \ + --hash=sha256:c8c751014e13d88d2be5f5f14fc8b89612fcfa92a9cc480f2bc1598357a23a05 \ + --hash=sha256:cad6b591a682dcc6cf1397c3900527f9affef1e55a06c4547264796bbd17cf5e \ + --hash=sha256:cbf8317bfccf0fed3b5680c559d3459cccf1abe9039bfa159e62e391c7270568 \ + --hash=sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96 \ + --hash=sha256:d169162803a24425eb5e4d51d79cbf429fd7a491b9e570a55f495ea55b26f0bf \ + --hash=sha256:d496e2f5245319da9d764296e86c5bb6fcf0cf7a8806d3d000717a889c8c0b7b \ + --hash=sha256:de987bb4e7ac95b99b805b99e0aae0ad51ae61df4263459d36e07cf4052d8b3a \ + --hash=sha256:df091cf961a3be783d6aebae963cc9b71e00d57fa6f149025075217bc6a55a7b \ + --hash=sha256:e99c7b90a29fd82fea9ef57943d501a16f3404d7b9ee81799d41639bdaae412c \ + --hash=sha256:eb844698d11433d2139bbeeb56499102143beb582bd6c194e3ba69c22f25c274 \ + --hash=sha256:f084813239e1eb403ddacd06a30de3d3e09a9b76e7894dcda2b22f8a726e9c60 \ + --hash=sha256:f25bbaf1235e27704f1a7b86cd3304eabc04f569c828101d94a0e605ef7205a5 \ + --hash=sha256:f65744d7a8bdb4bda5e1fa23e4ba16832860606fcc09d674d56e425e991539ec \ + --hash=sha256:f72fdbae2dbc6e68b8239defb48e6a5937b12218e6ffc2c7846cc37befa84362 # via uvicorn -httpx==0.28.1 +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad # via # huggingface-hub # litellm # mcp # openai -httpx-sse==0.4.3 +httpx-sse==0.4.3 \ + --hash=sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc \ + --hash=sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d # via mcp -huey==2.6.0 +huey==2.6.0 \ + --hash=sha256:1b9df9d370b49c6d5721ba8a01ac9a787cf86b3bdc584e4679de27b920395c3f \ + --hash=sha256:8d11f8688999d65266af1425b831f6e3773e99415027177b8734b0ffd5e251f6 # via mlflow -huggingface-hub==1.6.0 +huggingface-hub==1.6.0 \ + --hash=sha256:d931ddad8ba8dfc1e816bf254810eb6f38e5c32f60d4184b5885662a3b167325 \ + --hash=sha256:ef40e2d5cb85e48b2c067020fa5142168342d5108a1b267478ed384ecbf18961 # via tokenizers -idna==3.11 +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 # via # anyio # httpx # requests # yarl -importlib-metadata==8.7.1 +importlib-metadata==8.7.1 \ + --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \ + --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151 # via # litellm # mlflow-skinny # opentelemetry-api -itsdangerous==2.2.0 +itsdangerous==2.2.0 \ + --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \ + --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173 # via flask -jinja2==3.1.6 +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via # flask # flask-socketio # litellm -jiter==0.13.0 +jiter==0.13.0 \ + --hash=sha256:00203f47c214156df427b5989de74cb340c65c8180d09be1bf9de81d0abad599 \ + --hash=sha256:04670992b576fa65bd056dbac0c39fe8bd67681c380cb2b48efa885711d9d726 \ + --hash=sha256:0733312953b909688ae3c2d58d043aa040f9f1a6a75693defed7bc2cc4bf2654 \ + --hash=sha256:07b75fe09a4ee8e0c606200622e571e44943f47254f95e2436c8bdcaceb36d7d \ + --hash=sha256:0a2bd69fc1d902e89925fc34d1da51b2128019423d7b339a45d9e99c894e0663 \ + --hash=sha256:0a8d76c7524087272c8ae913f5d9d608bd839154b62c4322ef65723d2e5bb0b8 \ + --hash=sha256:0b34c519e17658ed88d5047999a93547f8889f3c1824120c26ad6be5f27b6cf5 \ + --hash=sha256:0bf670e3b1445fc4d31612199f1744f67f889ee1bbae703c4b54dc097e5dd394 \ + --hash=sha256:0c365005b05505a90d1c47856420980d0237adf82f70c4aff7aebd3c1cc143ad \ + --hash=sha256:0e3a5f0cde8ff433b8e88e41aa40131455420fb3649a3c7abdda6145f8cb7202 \ + --hash=sha256:0f0c065695f616a27c920a56ad0d4fc46415ef8b806bf8fc1cacf25002bd24e1 \ + --hash=sha256:1211427574b17b633cfceba5040de8081e5abf114f7a7602f73d2e16f9fdaa59 \ + --hash=sha256:1317fdffd16f5873e46ce27d0e0f7f4f90f0cdf1d86bf6abeaea9f63ca2c401d \ + --hash=sha256:15db60e121e11fe186c0b15236bd5d18381b9ddacdcf4e659feb96fc6c969c92 \ + --hash=sha256:19928b5d1ce0ff8c1ee1b9bdef3b5bfc19e8304f1b904e436caf30bc15dc6cf5 \ + --hash=sha256:19cd6f85e1dc090277c3ce90a5b7d96f32127681d825e71c9dce28788e39fc0c \ + --hash=sha256:1f4748aad1b4a93c8bdd70f604d0f748cdc0e8744c5547798acfa52f10e79228 \ + --hash=sha256:1f8a55b848cbabf97d861495cd65f1e5c590246fabca8b48e1747c4dfc8f85bf \ + --hash=sha256:2113c17c9a67071b0f820733c0893ed1d467b5fcf4414068169e5c2cabddb1e2 \ + --hash=sha256:24ab43126d5e05f3d53a36a8e11eb2f23304c6c1117844aaaf9a0aa5e40b5018 \ + --hash=sha256:24dc96eca9f84da4131cdf87a95e6ce36765c3b156fc9ae33280873b1c32d5f6 \ + --hash=sha256:2b4972c6df33731aac0742b64fd0d18e0a69bc7d6e03108ce7d40c85fd9e3e6d \ + --hash=sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024 \ + --hash=sha256:2d08c9475d48b92892583df9da592a0e2ac49bcd41fae1fec4f39ba6cf107820 \ + --hash=sha256:2ffc63785fd6c7977defe49b9824ae6ce2b2e2b77ce539bdaf006c26da06342e \ + --hash=sha256:309549b778b949d731a2f0e1594a3f805716be704a73bf3ad9a807eed5eb5721 \ + --hash=sha256:3097d665a27bc96fd9bbf7f86178037db139f319f785e4757ce7ccbf390db6c2 \ + --hash=sha256:36ebfbcffafb146d0e6ffb3e74d51e03d9c35ce7c625c8066cdbfc7b953bdc72 \ + --hash=sha256:3b3fb8c2053acaef8580809ac1d1f7481a0a0bdc012fd7f5d8b18fb696a5a089 \ + --hash=sha256:3d744a6061afba08dd7ae375dcde870cffb14429b7477e10f67e9e6d68772a0a \ + --hash=sha256:41f92313d17989102f3cb5dd533a02787cdb99454d494344b0361355da52fcb9 \ + --hash=sha256:4397ee562b9f69d283e5674445551b47a5e8076fdde75e71bfac5891113dc543 \ + --hash=sha256:45f6f8efb2f3b0603092401dc2df79fa89ccbc027aaba4174d2d4133ed661434 \ + --hash=sha256:47455245307e4debf2ce6c6e65a717550a0244231240dcf3b8f7d64e4c2f22f4 \ + --hash=sha256:4a638816427006c1e3f0013eb66d391d7a3acda99a7b0cf091eff4497ccea33a \ + --hash=sha256:5467696f6b827f1116556cb0db620440380434591e93ecee7fd14d1a491b6daa \ + --hash=sha256:57aab48f40be1db920a582b30b116fe2435d184f77f0e4226f546794cedd9cf0 \ + --hash=sha256:597245258e6ad085d064780abfb23a284d418d3e61c57362d9449c6c7317ee2d \ + --hash=sha256:5a1aff1fbdb803a376d4d22a8f63f8e7ccbce0b4890c26cc7af9e501ab339ef0 \ + --hash=sha256:5d9b34ad56761b3bf0fbe8f7e55468704107608512350962d3317ffd7a4382d5 \ + --hash=sha256:6207fc61c395b26fffdcf637a0b06b4326f35bfa93c6e92fe1a166a21aeb6731 \ + --hash=sha256:632bf7c1d28421c00dd8bbb8a3bac5663e1f57d5cd5ed962bce3c73bf62608e6 \ + --hash=sha256:66aa3e663840152d18cc8ff1e4faad3dd181373491b9cfdc6004b92198d67911 \ + --hash=sha256:682161a67adea11e3aae9038c06c8b4a9a71023228767477d683f69903ebc607 \ + --hash=sha256:6c26a424569a59140fb51160a56df13f438a2b0967365e987889186d5fc2f6f9 \ + --hash=sha256:6eeb7db8bc77dc20476bc2f7407a23dbe3d46d9cc664b166e3d474e1c1de4baa \ + --hash=sha256:701a1e77d1e593c1b435315ff625fd071f0998c5f02792038a5ca98899261b7d \ + --hash=sha256:775e10de3849d0631a97c603f996f518159272db00fdda0a780f81752255ee9d \ + --hash=sha256:7772115877c53f62beeb8fd853cab692dbc04374ef623b30f997959a4c0e7e95 \ + --hash=sha256:7b88d649135aca526da172e48083da915ec086b54e8e73a425ba50999468cc08 \ + --hash=sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19 \ + --hash=sha256:7beae3a3d3b5212d3a55d2961db3c292e02e302feb43fce6a3f7a31b90ea6dfe \ + --hash=sha256:7c26ad6967c9dcedf10c995a21539c3aa57d4abad7001b7a84f621a263a6b605 \ + --hash=sha256:7f90023f8f672e13ea1819507d2d21b9d2d1c18920a3b3a5f1541955a85b5504 \ + --hash=sha256:879e768938e7b49b5e90b7e3fecc0dbec01b8cb89595861fb39a8967c5220d09 \ + --hash=sha256:87ce0f14c6c08892b610686ae8be350bf368467b6acd5085a5b65441e2bf36d2 \ + --hash=sha256:8d76029f077379374cf0dbc78dbe45b38dec4a2eb78b08b5194ce836b2517afc \ + --hash=sha256:9621ca242547edc16400981ca3231e0c91c0c4c1ab8573a596cd9bb3575d5c2b \ + --hash=sha256:964538479359059a35fb400e769295d4b315ae61e4105396d355a12f7fef09f0 \ + --hash=sha256:9776ebe51713acf438fd9b4405fcd86893ae5d03487546dae7f34993217f8a91 \ + --hash=sha256:98fbafb6e88256f4454de33c1f40203d09fc33ed19162a68b3b257b29ca7f663 \ + --hash=sha256:9950290340acc1adaded363edd94baebcee7dabdfa8bee4790794cd5cfad2af6 \ + --hash=sha256:9d01ecc3a8cbdb6f25a37bd500510550b64ddf9f7d64a107d92f3ccb25035d0f \ + --hash=sha256:9da38b4fedde4fb528c740c2564628fbab737166a0e73d6d46cb4bb5463ff411 \ + --hash=sha256:9ffda299e417dc83362963966c50cb76d42da673ee140de8a8ac762d4bb2378b \ + --hash=sha256:a13b68cd1cd8cc9de8f244ebae18ccb3e4067ad205220ef324c39181e23bbf66 \ + --hash=sha256:a3a377af27b236abbf665a69b2bdd680e3b5a0bd2af825cd3b81245279a7606c \ + --hash=sha256:a576f5dce9ac7de5d350b8e2f552cf364f32975ed84717c35379a51c7cb198bd \ + --hash=sha256:a7637d92b1c9d7a771e8c56f445c7f84396d48f2e756e5978840ecba2fac0894 \ + --hash=sha256:ab1185ca5c8b9491b55ebf6c1e8866b8f68258612899693e24a92c5fdb9455d5 \ + --hash=sha256:ab44b178f7981fcaea7e0a5df20e773c663d06ffda0198f1a524e91b2fde7e59 \ + --hash=sha256:ade8cb6ff5632a62b7dbd4757d8c5573f7a2e9ae285d6b5b841707d8363205ef \ + --hash=sha256:aed40e099404721d7fcaf5b89bd3b4568a4666358bcac7b6b15c09fb6252ab68 \ + --hash=sha256:b1cbfa133241d0e6bdab48dcdc2604e8ba81512f6bbd68ec3e8e1357dd3c316c \ + --hash=sha256:b22945be8425d161f2e536cdae66da300b6b000f1c0ba3ddf237d1bfd45d21b8 \ + --hash=sha256:bb7613e1a427cfcb6ea4544f9ac566b93d5bf67e0d48c787eca673ff9c9dff2b \ + --hash=sha256:bcdabaea26cb04e25df3103ce47f97466627999260290349a88c8136ecae0060 \ + --hash=sha256:bdaba7d87e66f26a2c45d8cbadcbfc4bf7884182317907baf39cfe9775bb4d93 \ + --hash=sha256:c05b450d37ba0c9e21c77fef1f205f56bcee2330bddca68d344baebfc55ae0df \ + --hash=sha256:c1b609e5cbd2f52bb74fb721515745b407df26d7b800458bd97cb3b972c29e7d \ + --hash=sha256:c1e2b199f446d3e82246b4fd9236d7cb502dc2222b18698ba0d986d2fecc6152 \ + --hash=sha256:c3524798e70655ff19aec58c7d05adb1f074fecff62da857ea9be2b908b6d701 \ + --hash=sha256:cc5223ab19fe25e2f0bf2643204ad7318896fe3729bf12fde41b77bfc4fafff0 \ + --hash=sha256:d2a6394e6af690d462310a86b53c47ad75ac8c21dc79f120714ea449979cb1d3 \ + --hash=sha256:db367d8be9fad6e8ebbac4a7578b7af562e506211036cba2c06c3b998603c3d2 \ + --hash=sha256:dc3ce84cfd4fa9628fe62c4f85d0d597a4627d4242cfafac32a12cc1455d00f7 \ + --hash=sha256:e104da1db1c0991b3eaed391ccd650ae8d947eab1480c733e5a3fb28d4313e40 \ + --hash=sha256:e404ea551d35438013c64b4f357b0474c7abf9f781c06d44fcaf7a14c69ff9e2 \ + --hash=sha256:e5562a0f0e90a6223b704163ea28e831bd3a9faa3512a711f031611e6b06c939 \ + --hash=sha256:ea026e70a9a28ebbdddcbcf0f1323128a8db66898a06eaad3a4e62d2f554d096 \ + --hash=sha256:ec7e287d7fbd02cb6e22f9a00dd9c9cd504c40a61f2c61e7e1f9690a82726b4c \ + --hash=sha256:ed0240dd1536a98c3ab55e929c60dfff7c899fecafcb7d01161b21a99fc8c363 \ + --hash=sha256:ed9bbc30f5d60a3bdf63ae76beb3f9db280d7f195dfcfa61af792d6ce912d159 \ + --hash=sha256:ee9da221dca6e0429c2704c1b3655fe7b025204a71d4d9b73390c759d776d165 \ + --hash=sha256:f22ef501c3f87ede88f23f9b11e608581c14f04db59b6a801f354397ae13739f \ + --hash=sha256:f2839f9c2c7e2dffc1bc5929a510e14ce0a946be9365fd1219e7ef342dae14f4 \ + --hash=sha256:f556aa591c00f2c45eb1b89f68f52441a016034d18b65da60e2d2875bbbf344a \ + --hash=sha256:f7e1d61da332ec412350463891923f960c3073cf1aae93b538f0bb4c8cd46efb \ + --hash=sha256:f917a04240ef31898182f76a332f508f2cc4b57d2b4d7ad2dbfebbfe167eb505 \ + --hash=sha256:fa476ab5dd49f3bf3a168e05f89358c75a17608dbabb080ef65f96b27c19ab10 \ + --hash=sha256:fe49d3ff6db74321f144dff9addd4a5874d3105ac5ba7c5b77fac099cfae31ae \ + --hash=sha256:ff732bd0a0e778f43d5009840f20b935e79087b4dc65bd36f1cd0f9b04b8ff7f # via openai -jmespath==1.1.0 +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 # via # boto3 # botocore -joblib==1.5.3 +joblib==1.5.3 \ + --hash=sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713 \ + --hash=sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3 # via scikit-learn -jsonschema==4.26.0 +jsonschema==4.26.0 \ + --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce # via # litellm # mcp -jsonschema-specifications==2025.9.1 +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ + --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d # via jsonschema -kiwisolver==1.5.0 +kiwisolver==1.5.0 \ + --hash=sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9 \ + --hash=sha256:01808c6d15f4c3e8559595d6d1fe6411c68e4a3822b4b9972b44473b24f4e679 \ + --hash=sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0 \ + --hash=sha256:0b85aad90cea8ac6797a53b5d5f2e967334fa4d1149f031c4537569972596cb8 \ + --hash=sha256:0bf3acf1419fa93064a4c2189ac0b58e3be7872bf6ee6177b0d4c63dc4cea276 \ + --hash=sha256:0c50b89ffd3e1a911c69a1dd3de7173c0cd10b130f56222e57898683841e4f96 \ + --hash=sha256:0cbe94b69b819209a62cb27bdfa5dc2a8977d8de2f89dfd97ba4f53ed3af754e \ + --hash=sha256:0df54df7e686afa55e6f21fb86195224a6d9beb71d637e8d7920c95cf0f89aac \ + --hash=sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f \ + --hash=sha256:12e91c215a96e39f57989c8912ae761286ac5a9584d04030ceb3368a357f017a \ + --hash=sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15 \ + --hash=sha256:16b85d37c2cbb3253226d26e64663f755d88a03439a9c47df6246b35defbdfb7 \ + --hash=sha256:1b0feb50971481a2cc44d94e88bdb02cdd497618252ae226b8eb1201b957e368 \ + --hash=sha256:1d49a49ac4cbfb7c1375301cd1ec90169dfeae55ff84710d782260ce77a75a02 \ + --hash=sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9 \ + --hash=sha256:1dd9b0b119a350976a6d781e7278ec7aca0b201e1a9e2d23d9804afecb6ca681 \ + --hash=sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57 \ + --hash=sha256:2517e24d7315eb51c10664cdb865195df38ab74456c677df67bb47f12d088a27 \ + --hash=sha256:295d9ffe712caa9f8a3081de8d32fc60191b4b51c76f02f951fd8407253528f4 \ + --hash=sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920 \ + --hash=sha256:32cc0a5365239a6ea0c6ed461e8838d053b57e397443c0ca894dcc8e388d4374 \ + --hash=sha256:332b4f0145c30b5f5ad9374881133e5aa64320428a57c2c2b61e9d891a51c2f3 \ + --hash=sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa \ + --hash=sha256:38f4a703656f493b0ad185211ccfca7f0386120f022066b018eb5296d8613e23 \ + --hash=sha256:3ac2360e93cb41be81121755c6462cff3beaa9967188c866e5fce5cf13170859 \ + --hash=sha256:3c4923e404d6bcd91b6779c009542e5647fef32e4a5d75e115e3bbac6f2335eb \ + --hash=sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d \ + --hash=sha256:41024ed50e44ab1a60d3fe0a9d15a4ccc9f5f2b1d814ff283c8d01134d5b81bc \ + --hash=sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581 \ + --hash=sha256:4432b835675f0ea7414aab3d37d119f7226d24869b7a829caeab49ebda407b0c \ + --hash=sha256:4db576bb8c3ef9365f8b40fe0f671644de6736ae2c27a2c62d7d8a1b4329f099 \ + --hash=sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05 \ + --hash=sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9 \ + --hash=sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd \ + --hash=sha256:50847dca5d197fcbd389c805aa1a1cf32f25d2e7273dc47ab181a517666b68cc \ + --hash=sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796 \ + --hash=sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303 \ + --hash=sha256:51e8c4084897de9f05898c2c2a39af6318044ae969d46ff7a34ed3f96274adca \ + --hash=sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314 \ + --hash=sha256:56fa888f10d0f367155e76ce849fa1166fc9730d13bd2d65a2aa13b6f5424489 \ + --hash=sha256:58f812017cd2985c21fbffb4864d59174d4903dd66fa23815e74bbc7a0e2dd57 \ + --hash=sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1 \ + --hash=sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797 \ + --hash=sha256:5b233ea3e165e43e35dba1d2b8ecc21cf070b45b65ae17dd2747d2713d942021 \ + --hash=sha256:6176c1811d9d5a04fa391c490cc44f451e240697a16977f11c6f722efb9041db \ + --hash=sha256:62f59da443c4f4849f73a51a193b1d9d258dcad0c41bc4d1b8fb2bcc04bfeb22 \ + --hash=sha256:6783e069732715ad0c3ce96dbf21dbc2235ab0593f2baf6338101f70371f4028 \ + --hash=sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083 \ + --hash=sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65 \ + --hash=sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588 \ + --hash=sha256:7a32f72973f0f950c1920475d5c5ea3d971b81b6f0ec53b8d0a956cc965f22e0 \ + --hash=sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a \ + --hash=sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1 \ + --hash=sha256:800ee55980c18545af444d93fdd60c56b580db5cc54867d8cbf8a1dc0829938c \ + --hash=sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac \ + --hash=sha256:86e0287879f75621ae85197b0877ed2f8b7aa57b511c7331dce2eb6f4de7d476 \ + --hash=sha256:893ff3a711d1b515ba9da14ee090519bad4610ed1962fbe298a434e8c5f8db53 \ + --hash=sha256:89fc958c702ee9a745e4700378f5d23fddbc46ff89e8fdbf5395c24d5c1452a3 \ + --hash=sha256:8c63c91f95173f9c2a67c7c526b2cea976828a0e7fced9cdcead2802dc10f8a4 \ + --hash=sha256:8df31fe574b8b3993cc61764f40941111b25c2d9fea13d3ce24a49907cd2d615 \ + --hash=sha256:8f9baf6f0a6e7571c45c8863010b45e837c3ee1c2c77fcd6ef423be91b21fedb \ + --hash=sha256:9027d773c4ff81487181a925945743413f6069634d0b122d0b37684ccf4f1e18 \ + --hash=sha256:9190426b7aa26c5229501fa297b8d0653cfd3f5a36f7990c264e157cbf886b3b \ + --hash=sha256:940dda65d5e764406b9fb92761cbf462e4e63f712ab60ed98f70552e496f3bf1 \ + --hash=sha256:94eff26096eb5395136634622515b234ecb6c9979824c1f5004c6e3c3c85ccd2 \ + --hash=sha256:9eed0f7edbb274413b6ee781cca50541c8c0facd3d6fd289779e494340a2b85c \ + --hash=sha256:ad4ae4ffd1ee9cd11357b4c66b612da9888f4f4daf2f36995eda64bd45370cac \ + --hash=sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d \ + --hash=sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf \ + --hash=sha256:b7d335370ae48a780c6e6a6bbfa97342f563744c39c35562f3f367665f5c1de2 \ + --hash=sha256:b83af57bdddef03c01a9138034c6ff03181a3028d9a1003b301eb1a55e161a3f \ + --hash=sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f \ + --hash=sha256:bc4d8e252f532ab46a1de9349e2d27b91fce46736a9eedaa37beaca66f574ed4 \ + --hash=sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9 \ + --hash=sha256:be12f931839a3bdfe28b584db0e640a65a8bcbc24560ae3fdb025a449b3d754e \ + --hash=sha256:be4a51a55833dc29ab5d7503e7bcb3b3af3402d266018137127450005cdfe737 \ + --hash=sha256:beb7f344487cdcb9e1efe4b7a29681b74d34c08f0043a327a74da852a6749e7b \ + --hash=sha256:bf4679a3d71012a7c2bf360e5cd878fbd5e4fcac0896b56393dec239d81529ed \ + --hash=sha256:c0e1403fd7c26d77c1f03e096dc58a5c726503fa0db0456678b8668f76f521e3 \ + --hash=sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7 \ + --hash=sha256:c438f6ca858697c9ab67eb28246c92508af972e114cac34e57a6d4ba17a3ac08 \ + --hash=sha256:c8277104ded0a51e699c8c3aff63ce2c56d4ed5519a5f73e0fd7057f959a2b9e \ + --hash=sha256:c95cab08d1965db3d84a121f1c7ce7479bdd4072c9b3dafd8fecce48a2e6b902 \ + --hash=sha256:cc0b66c1eec9021353a4b4483afb12dfd50e3669ffbb9152d6842eb34c7e29fd \ + --hash=sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6 \ + --hash=sha256:ce9bf03dad3b46408c08649c6fbd6ca28a9fce0eb32fdfffa6775a13103b5310 \ + --hash=sha256:cff8e5383db4989311f99e814feeb90c4723eb4edca425b9d5d9c3fefcdd9537 \ + --hash=sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554 \ + --hash=sha256:d1ffeb80b5676463d7a7d56acbe8e37a20ce725570e09549fe738e02ca6b7e1e \ + --hash=sha256:d36ca54cb4c6c4686f7cbb7b817f66f5911c12ddb519450bbe86707155028f87 \ + --hash=sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a \ + --hash=sha256:d5cd5189fc2b6a538b75ae45433140c4823463918f7b1617c31e68b085c0022c \ + --hash=sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79 \ + --hash=sha256:d76e2d8c75051d58177e762164d2e9ab92886534e3a12e795f103524f221dd8e \ + --hash=sha256:daae526907e262de627d8f70058a0f64acc9e2641c164c99c8f594b34a799a16 \ + --hash=sha256:db485b3847d182b908b483b2ed133c66d88d49cacf98fd278fadafe11b4478d1 \ + --hash=sha256:dd952e03bfbb096cfe2dd35cd9e00f269969b67536cb4370994afc20ff2d0875 \ + --hash=sha256:dda366d548e89a90d88a86c692377d18d8bd64b39c1fb2b92cb31370e2896bbd \ + --hash=sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0 \ + --hash=sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9 \ + --hash=sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646 \ + --hash=sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657 \ + --hash=sha256:ebae99ed6764f2b5771c522477b311be313e8841d2e0376db2b10922daebbba4 \ + --hash=sha256:ec4c85dc4b687c7f7f15f553ff26a98bfe8c58f5f7f0ac8905f0ba4c7be60232 \ + --hash=sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819 \ + --hash=sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384 \ + --hash=sha256:f1f9f4121ec58628c96baa3de1a55a4e3a333c5102c8e94b64e23bf7b2083309 \ + --hash=sha256:f42c23db5d1521218a3276bb08666dcb662896a0be7347cba864eca45ff64ede \ + --hash=sha256:f443b4825c50a51ee68585522ab4a1d1257fac65896f282b4c6763337ac9f5d2 \ + --hash=sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203 \ + --hash=sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7 \ + --hash=sha256:fa6248cd194edff41d7ea9425ced8ca3a6f838bfb295f6f1d6e6bb694a8518df \ + --hash=sha256:fa8eb9ecdb7efb0b226acec134e0d709e87a909fa4971a54c0c4f6e88635484c \ + --hash=sha256:fc20894c3d21194d8041a28b65622d5b86db786da6e3cfe73f0c762951a61167 \ + --hash=sha256:fc4d3f1fb9ca0ae9f97b095963bc6326f1dbfd3779d6679a1e016b9baaa153d3 \ + --hash=sha256:fd40bb9cd0891c4c3cb1ddf83f8bbfa15731a248fdc8162669405451e2724b09 \ + --hash=sha256:ff710414307fefa903e0d9bdf300972f892c23477829f49504e59834f4195398 # via matplotlib -limits==5.8.0 +limits==5.8.0 \ + --hash=sha256:ae1b008a43eb43073c3c579398bd4eb4c795de60952532dc24720ab45e1ac6b8 \ + --hash=sha256:c9e0d74aed837e8f6f50d1fcebcf5fd8130957287206bc3799adaee5092655da # via slowapi -litellm==1.82.1 +litellm==1.82.1 \ + --hash=sha256:a9ec3fe42eccb1611883caaf8b1bf33c9f4e12163f94c7d1004095b14c379eb2 \ + --hash=sha256:bc8427cdccc99e191e08e36fcd631c93b27328d1af789839eb3ac01a7d281890 # via mlflow -mako==1.3.10 +mako==1.3.10 \ + --hash=sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28 \ + --hash=sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59 # via alembic -markdown-it-py==4.0.0 +markdown-it-py==4.0.0 \ + --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \ + --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3 # via rich -markupsafe==3.0.3 +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 # via # flask # jinja2 # mako # werkzeug -matplotlib==3.10.8 +matplotlib==3.10.8 \ + --hash=sha256:00270d217d6b20d14b584c521f810d60c5c78406dc289859776550df837dcda7 \ + --hash=sha256:0a33deb84c15ede243aead39f77e990469fff93ad1521163305095b77b72ce4a \ + --hash=sha256:113bb52413ea508ce954a02c10ffd0d565f9c3bc7f2eddc27dfe1731e71c7b5f \ + --hash=sha256:12d90df9183093fcd479f4172ac26b322b1248b15729cb57f42f71f24c7e37a3 \ + --hash=sha256:15d30132718972c2c074cd14638c7f4592bd98719e2308bccea40e0538bc0cb5 \ + --hash=sha256:18821ace09c763ec93aef5eeff087ee493a24051936d7b9ebcad9662f66501f9 \ + --hash=sha256:1ae029229a57cd1e8fe542485f27e7ca7b23aa9e8944ddb4985d0bc444f1eca2 \ + --hash=sha256:2299372c19d56bcd35cf05a2738308758d32b9eaed2371898d8f5bd33f084aa3 \ + --hash=sha256:238b7ce5717600615c895050239ec955d91f321c209dd110db988500558e70d6 \ + --hash=sha256:24d50994d8c5816ddc35411e50a86ab05f575e2530c02752e02538122613371f \ + --hash=sha256:25d380fe8b1dc32cf8f0b1b448470a77afb195438bafdf1d858bfb876f3edf7b \ + --hash=sha256:2c1998e92cd5999e295a731bcb2911c75f597d937341f3030cc24ef2733d78a8 \ + --hash=sha256:2cf5bd12cecf46908f286d7838b2abc6c91cda506c0445b8223a7c19a00df008 \ + --hash=sha256:32f8dce744be5569bebe789e46727946041199030db8aeb2954d26013a0eb26b \ + --hash=sha256:37b3c1cc42aa184b3f738cfa18c1c1d72fd496d85467a6cf7b807936d39aa656 \ + --hash=sha256:3a48a78d2786784cc2413e57397981fb45c79e968d99656706018d6e62e57958 \ + --hash=sha256:3ab4aabc72de4ff77b3ec33a6d78a68227bf1123465887f9905ba79184a1cc04 \ + --hash=sha256:3c624e43ed56313651bc18a47f838b60d7b8032ed348911c54906b130b20071b \ + --hash=sha256:3f2e409836d7f5ac2f1c013110a4d50b9f7edc26328c108915f9075d7d7a91b6 \ + --hash=sha256:3f5c3e4da343bba819f0234186b9004faba952cc420fbc522dc4e103c1985908 \ + --hash=sha256:41703cc95688f2516b480f7f339d8851a6035f18e100ee6a32bc0b8536a12a9c \ + --hash=sha256:495672de149445ec1b772ff2c9ede9b769e3cb4f0d0aa7fa730d7f59e2d4e1c1 \ + --hash=sha256:4cf267add95b1c88300d96ca837833d4112756045364f5c734a2276038dae27d \ + --hash=sha256:56271f3dac49a88d7fca5060f004d9d22b865f743a12a23b1e937a0be4818ee1 \ + --hash=sha256:595ba4d8fe983b88f0eec8c26a241e16d6376fe1979086232f481f8f3f67494c \ + --hash=sha256:5f62550b9a30afde8c1c3ae450e5eb547d579dd69b25c2fc7a1c67f934c1717a \ + --hash=sha256:646d95230efb9ca614a7a594d4fcacde0ac61d25e37dd51710b36477594963ce \ + --hash=sha256:64fcc24778ca0404ce0cb7b6b77ae1f4c7231cdd60e6778f999ee05cbd581b9a \ + --hash=sha256:6be43b667360fef5c754dda5d25a32e6307a03c204f3c0fc5468b78fa87b4160 \ + --hash=sha256:6da7c2ce169267d0d066adcf63758f0604aa6c3eebf67458930f9d9b79ad1db1 \ + --hash=sha256:83d282364ea9f3e52363da262ce32a09dfe241e4080dcedda3c0db059d3c1f11 \ + --hash=sha256:9153c3292705be9f9c64498a8872118540c3f4123d1a1c840172edf262c8be4a \ + --hash=sha256:99eefd13c0dc3b3c1b4d561c1169e65fe47aab7b8158754d7c084088e2329466 \ + --hash=sha256:a0a7f52498f72f13d4a25ea70f35f4cb60642b466cbb0a9be951b5bc3f45a486 \ + --hash=sha256:a2b336e2d91a3d7006864e0990c83b216fcdca64b5a6484912902cef87313d78 \ + --hash=sha256:a48f2b74020919552ea25d222d5cc6af9ca3f4eb43a93e14d068457f545c2a17 \ + --hash=sha256:ad3d9833a64cf48cc4300f2b406c3d0f4f4724a91c0bd5640678a6ba7c102077 \ + --hash=sha256:b44d07310e404ba95f8c25aa5536f154c0a8ec473303535949e52eb71d0a1565 \ + --hash=sha256:b53285e65d4fa4c86399979e956235deb900be5baa7fc1218ea67fbfaeaadd6f \ + --hash=sha256:b5a2b97dbdc7d4f353ebf343744f1d1f1cca8aa8bfddb4262fcf4306c3761d50 \ + --hash=sha256:b9a5ca4ac220a0cdd1ba6bcba3608547117d30468fefce49bb26f55c1a3d5c58 \ + --hash=sha256:bab485bcf8b1c7d2060b4fcb6fc368a9e6f4cd754c9c2fea281f4be21df394a2 \ + --hash=sha256:c108a1d6fa78a50646029cb6d49808ff0fc1330fda87fa6f6250c6b5369b6645 \ + --hash=sha256:d56a1efd5bfd61486c8bc968fa18734464556f0fb8e51690f4ac25d85cbbbbc2 \ + --hash=sha256:d9050fee89a89ed57b4fb2c1bfac9a3d0c57a0d55aed95949eedbc42070fea39 \ + --hash=sha256:dd80ecb295460a5d9d260df63c43f4afbdd832d725a531f008dad1664f458adf \ + --hash=sha256:e8ea3e2d4066083e264e75c829078f9e149fa119d27e19acd503de65e0b13149 \ + --hash=sha256:eb3823f11823deade26ce3b9f40dcb4a213da7a670013929f31d5f5ed1055b22 \ + --hash=sha256:ee40c27c795bda6a5292e9cff9890189d32f7e3a0bf04e0e3c9430c4a00c37df \ + --hash=sha256:efb30e3baaea72ce5928e32bab719ab4770099079d66726a62b11b1ef7273be4 \ + --hash=sha256:f254d118d14a7f99d616271d6c3c27922c092dac11112670b157798b89bf4933 \ + --hash=sha256:f89c151aab2e2e23cb3fe0acad1e8b82841fd265379c4cecd0f3fcb34c15e0f6 \ + --hash=sha256:f97aeb209c3d2511443f8797e3e5a569aebb040d4f8bc79aa3ee78a8fb9e3dd8 \ + --hash=sha256:f9b587c9c7274c1613a30afabf65a272114cd6cdbe67b3406f818c79d7ab2e2a \ + --hash=sha256:fb061f596dad3a0f52b60dc6a5dec4a0c300dec41e058a7efe09256188d170b7 # via mlflow -mcp==1.26.0 +mcp==1.26.0 \ + --hash=sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca \ + --hash=sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66 # via claude-agent-sdk -mdurl==0.1.2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -mlflow==3.10.1 +mlflow==3.10.1 \ + --hash=sha256:17bfbd76d4071498d6199c3fc53945e5f50997d14e3e2a6bfd4dc3cb8957f209 \ + --hash=sha256:609509ccc15eb9c17861748e537cbffa57d2caf488ff3e30efed62951a6977cf # via -r requirements.txt -mlflow-skinny==3.10.1 +mlflow-skinny==3.10.1 \ + --hash=sha256:3d1c5c30245b6e7065b492b09dd47be7528e0a14c4266b782fe58f9bcd1e0be0 \ + --hash=sha256:df1dd507d8ddadf53bfab2423c76cdcafc235cd1a46921a06d1a6b4dd04b023c # via mlflow -mlflow-tracing==3.10.1 +mlflow-tracing==3.10.1 \ + --hash=sha256:649c722cc58d54f1f40559023a6bd6f3f08150c3ce3c3bb27972b3e795890f47 \ + --hash=sha256:9e54d63cf776d29bb9e2278d35bf27352b93f7b35c8fe8452e9ba5e2a3c5b78f # via mlflow -multidict==6.7.1 +multidict==6.7.1 \ + --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ + --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ + --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \ + --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \ + --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \ + --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \ + --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \ + --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \ + --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \ + --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \ + --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \ + --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \ + --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \ + --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \ + --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \ + --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \ + --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \ + --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \ + --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \ + --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \ + --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \ + --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \ + --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \ + --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \ + --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \ + --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \ + --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \ + --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \ + --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \ + --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \ + --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \ + --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \ + --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \ + --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \ + --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \ + --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \ + --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \ + --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \ + --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \ + --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \ + --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \ + --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \ + --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \ + --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \ + --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \ + --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \ + --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \ + --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \ + --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \ + --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \ + --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \ + --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \ + --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \ + --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \ + --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \ + --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \ + --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \ + --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \ + --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \ + --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \ + --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \ + --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \ + --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \ + --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \ + --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \ + --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \ + --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \ + --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \ + --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \ + --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \ + --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \ + --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \ + --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \ + --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \ + --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \ + --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \ + --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \ + --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \ + --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \ + --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \ + --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \ + --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \ + --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \ + --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \ + --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \ + --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \ + --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \ + --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \ + --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \ + --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \ + --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \ + --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \ + --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \ + --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \ + --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \ + --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \ + --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \ + --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \ + --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \ + --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \ + --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \ + --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \ + --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \ + --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \ + --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \ + --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \ + --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \ + --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \ + --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \ + --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \ + --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \ + --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \ + --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \ + --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \ + --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \ + --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \ + --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \ + --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \ + --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \ + --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \ + --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \ + --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \ + --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \ + --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \ + --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \ + --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \ + --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \ + --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \ + --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \ + --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \ + --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \ + --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \ + --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \ + --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \ + --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \ + --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \ + --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \ + --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \ + --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \ + --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \ + --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \ + --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \ + --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \ + --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \ + --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \ + --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19 # via # aiohttp # yarl -numpy==2.4.3 +numpy==2.4.3 \ + --hash=sha256:0200b25c687033316fb39f0ff4e3e690e8957a2c3c8d22499891ec58c37a3eb5 \ + --hash=sha256:0448e7f9caefb34b4b7dd2b77f21e8906e5d6f0365ad525f9f4f530b13df2afc \ + --hash=sha256:0a195f4216be9305a73c0e91c9b026a35f2161237cf1c6de9b681637772ea657 \ + --hash=sha256:0a60e17a14d640f49146cb38e3f105f571318db7826d9b6fef7e4dce758faecd \ + --hash=sha256:120df8c0a81ebbf5b9020c91439fccd85f5e018a927a39f624845be194a2be02 \ + --hash=sha256:148d59127ac95979d6f07e4d460f934ebdd6eed641db9c0db6c73026f2b2101a \ + --hash=sha256:1ec84fd7c8e652b0f4aaaf2e6e9cc8eaa9b1b80a537e06b2e3a2fb176eedcb26 \ + --hash=sha256:22654fe6be0e5206f553a9250762c653d3698e46686eee53b399ab90da59bd92 \ + --hash=sha256:22c31dc07025123aedf7f2db9e91783df13f1776dc52c6b22c620870dc0fab22 \ + --hash=sha256:23b46bb6d8ecb68b58c09944483c135ae5f0e9b8d8858ece5e4ead783771d2a9 \ + --hash=sha256:2629289168f4897a3c4e23dc98d6f1731f0fc0fe52fb9db19f974041e4cc12b9 \ + --hash=sha256:26952e18d82a1dbbc2f008d402021baa8d6fc8e84347a2072a25e08b46d698b9 \ + --hash=sha256:29363fbfa6f8ee855d7569c96ce524845e3d726d6c19b29eceec7dd555dab152 \ + --hash=sha256:297837823f5bc572c5f9379b0c9f3a3365f08492cbdc33bcc3af174372ebb168 \ + --hash=sha256:2abad5c7fef172b3377502bde47892439bae394a71bc329f31df0fd829b41a9e \ + --hash=sha256:2b3f8d2c4589b1a2028d2a770b0fc4d1f332fb5e01521f4de3199a896d158ddd \ + --hash=sha256:2ddb7919366ee468342b91dea2352824c25b55814a987847b6c52003a7c97f15 \ + --hash=sha256:2e03c05abaee1f672e9d67bc858f300b5ccba1c21397211e8d77d98350972093 \ + --hash=sha256:32e3bef222ad6b052280311d1d60db8e259e4947052c3ae7dd6817451fc8a4c5 \ + --hash=sha256:33b3bf58ee84b172c067f56aeadc7ee9ab6de69c5e800ab5b10295d54c581adb \ + --hash=sha256:45f003dbdffb997a03da2d1d0cb41fbd24a87507fb41605c0420a3db5bd4667b \ + --hash=sha256:483a201202b73495f00dbc83796c6ae63137a9bdade074f7648b3e32613412dd \ + --hash=sha256:48da3a4ee1336454b07497ff7ec83903efa5505792c4e6d9bf83d99dc07a1e18 \ + --hash=sha256:4b42639cdde6d24e732ff823a3fa5b701d8acad89c4142bc1d0bd6dc85200ba5 \ + --hash=sha256:4bd4741a6a676770e0e97fe9ab2e51de01183df3dcbcec591d26d331a40de950 \ + --hash=sha256:4d382735cecd7bcf090172489a525cd7d4087bc331f7df9f60ddc9a296cf208e \ + --hash=sha256:52077feedeff7c76ed7c9f1a0428558e50825347b7545bbb8523da2cd55c547a \ + --hash=sha256:54f29b877279d51e210e0c80709ee14ccbbad647810e8f3d375561c45ef613dd \ + --hash=sha256:5884ce5c7acfae1e4e1b6fde43797d10aa506074d25b531b4f54bde33c0c31d4 \ + --hash=sha256:5e10da9e93247e554bb1d22f8edc51847ddd7dde52d85ce31024c1b4312bfba0 \ + --hash=sha256:61b0cbabbb6126c8df63b9a3a0c4b1f44ebca5e12ff6997b80fcf267fb3150ef \ + --hash=sha256:65f3c2455188f09678355f5cae1f959a06b778bc66d535da07bf2ef20cd319d5 \ + --hash=sha256:679f2a834bae9020f81534671c56fd0cc76dd7e5182f57131478e23d0dc59e24 \ + --hash=sha256:6bd06731541f89cdc01b261ba2c9e037f1543df7472517836b78dfb15bd6e476 \ + --hash=sha256:715de7f82e192e8cae5a507a347d97ad17598f8e026152ca97233e3666daaa71 \ + --hash=sha256:737f630a337364665aba3b5a77e56a68cc42d350edd010c345d65a3efa3addcc \ + --hash=sha256:7395e69ff32526710748f92cd8c9849b361830968ea3e24a676f272653e8983e \ + --hash=sha256:76dbb9d4e43c16cf9aa711fcd8de1e2eeb27539dcefb60a1d5e9f12fae1d1ed8 \ + --hash=sha256:76f0f283506c28b12bba319c0fab98217e9f9b54e6160e9c79e9f7348ba32e9c \ + --hash=sha256:77e76d932c49a75617c6d13464e41203cd410956614d0a0e999b25e9e8d27eec \ + --hash=sha256:7aa4e54f6469300ebca1d9eb80acd5253cdfa36f2c03d79a35883687da430875 \ + --hash=sha256:7d1ce23cce91fcea443320a9d0ece9b9305d4368875bab09538f7a5b4131938a \ + --hash=sha256:7e58765ad74dcebd3ef0208a5078fba32dc8ec3578fe84a604432950cd043d79 \ + --hash=sha256:7f3408ff897f8ab07a07fbe2823d7aee6ff644c097cc1f90382511fe982f647f \ + --hash=sha256:8ba7b51e71c05aa1f9bc3641463cd82308eab40ce0d5c7e1fd4038cbf9938147 \ + --hash=sha256:8e236dbda4e1d319d681afcbb136c0c4a8e0f1a5c58ceec2adebb547357fe857 \ + --hash=sha256:94f3c4a151a2e529adf49c1d54f0f57ff8f9b233ee4d44af623a81553ab86368 \ + --hash=sha256:9684823a78a6cd6ad7511fc5e25b07947d1d5b5e2812c93fe99d7d4195130720 \ + --hash=sha256:a016db5c5dba78fa8fe9f5d80d6708f9c42ab087a739803c0ac83a43d686a470 \ + --hash=sha256:a111698b4a3f8dcbe54c64a7708f049355abd603e619013c346553c1fd4ca90b \ + --hash=sha256:a1988292870c7cb9d0ebb4cc96b4d447513a9644801de54606dc7aabf2b7d920 \ + --hash=sha256:a315e5234d88067f2d97e1f2ef670a7569df445d55400f1e33d117418d008d52 \ + --hash=sha256:a749547700de0a20a6718293396ec237bb38218049cfce788e08fcb716e8cf73 \ + --hash=sha256:a97cbf7e905c435865c2d939af3d93f99d18eaaa3cabe4256f4304fb51604349 \ + --hash=sha256:abdce0f71dcb4a00e4e77f3faf05e4616ceccfe72ccaa07f47ee79cda3b7b0f4 \ + --hash=sha256:b346845443716c8e542d54112966383b448f4a3ba5c66409771b8c0889485dd3 \ + --hash=sha256:b44fd60341c4d9783039598efadd03617fa28d041fc37d22b62d08f2027fa0e7 \ + --hash=sha256:bb2e3cf95854233799013779216c57e153c1ee67a0bf92138acca0e429aefaee \ + --hash=sha256:bc71942c789ef415a37f0d4eab90341425a00d538cd0642445d30b41023d3395 \ + --hash=sha256:be3b8487d725a77acccc9924f65fd8bce9af7fac8c9820df1049424a2115af6c \ + --hash=sha256:c59020932feb24ed49ffd03704fbab89f22aa9c0d4b180ff45542fe8918f5611 \ + --hash=sha256:c6b124bfcafb9e8d3ed09130dbee44848c20b3e758b6bbf006e641778927c028 \ + --hash=sha256:c9619741e9da2059cd9c3f206110b97583c7152c1dc9f8aafd4beb450ac1c89d \ + --hash=sha256:cd32fbacb9fd1bf041bf8e89e4576b6f00b895f06d00914820ae06a616bdfef7 \ + --hash=sha256:d1b90d840b25874cf5cd20c219af10bac3667db3876d9a495609273ebe679070 \ + --hash=sha256:d213c7e6e8d211888cc359bab7199670a00f5b82c0978b9d1c75baf1eddbeac0 \ + --hash=sha256:d5f51900414fc9204a0e0da158ba2ac52b75656e7dce7e77fb9f84bfa343b4cc \ + --hash=sha256:d71e379452a2f670ccb689ec801b1218cd3983e253105d6e83780967e899d687 \ + --hash=sha256:d84f0f881cb2225c2dfd7f78a10a5645d487a496c6668d6cc39f0f114164f3d0 \ + --hash=sha256:decb0eb8a53c3b009b0962378065589685d66b23467ef5dac16cbe818afde27f \ + --hash=sha256:e7dd01a46700b1967487141a66ac1a3cf0dd8ebf1f08db37d46389401512ca97 \ + --hash=sha256:eb610595dd91560905c132c709412b512135a60f1851ccbd2c959e136431ff67 # via # contourpy # matplotlib @@ -231,33 +1758,49 @@ numpy==2.4.3 # scikit-learn # scipy # skops -openai==2.26.0 +openai==2.26.0 \ + --hash=sha256:6151bf8f83802f036117f06cc8a57b3a4da60da9926826cc96747888b57f394f \ + --hash=sha256:b41f37c140ae0034a6e92b0c509376d907f3a66109935fba2c1b471a7c05a8fb # via litellm -opentelemetry-api==1.40.0 +opentelemetry-api==1.40.0 \ + --hash=sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f \ + --hash=sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9 # via # mlflow-skinny # mlflow-tracing # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp-proto-common==1.40.0 +opentelemetry-exporter-otlp-proto-common==1.40.0 \ + --hash=sha256:1cbee86a4064790b362a86601ee7934f368b81cd4cc2f2e163902a6e7818a0fa \ + --hash=sha256:7081ff453835a82417bf38dccf122c827c3cbc94f2079b03bba02a3165f25149 # via opentelemetry-exporter-otlp-proto-grpc -opentelemetry-exporter-otlp-proto-grpc==1.40.0 +opentelemetry-exporter-otlp-proto-grpc==1.40.0 \ + --hash=sha256:2aa0ca53483fe0cf6405087a7491472b70335bc5c7944378a0a8e72e86995c52 \ + --hash=sha256:bd4015183e40b635b3dab8da528b27161ba83bf4ef545776b196f0fb4ec47740 # via -r requirements.txt -opentelemetry-proto==1.40.0 +opentelemetry-proto==1.40.0 \ + --hash=sha256:03f639ca129ba513f5819810f5b1f42bcb371391405d99c168fe6937c62febcd \ + --hash=sha256:266c4385d88923a23d63e353e9761af0f47a6ed0d486979777fe4de59dc9b25f # via # mlflow-skinny # mlflow-tracing # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc -opentelemetry-sdk==1.40.0 +opentelemetry-sdk==1.40.0 \ + --hash=sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2 \ + --hash=sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1 # via # mlflow-skinny # mlflow-tracing # opentelemetry-exporter-otlp-proto-grpc -opentelemetry-semantic-conventions==0.61b0 +opentelemetry-semantic-conventions==0.61b0 \ + --hash=sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a \ + --hash=sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2 # via opentelemetry-sdk -packaging==26.0 +packaging==26.0 \ + --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ + --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 # via # gunicorn # huggingface-hub @@ -266,34 +1809,372 @@ packaging==26.0 # mlflow-skinny # mlflow-tracing # skops -pandas==2.3.3 +pandas==2.3.3 \ + --hash=sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7 \ + --hash=sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593 \ + --hash=sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5 \ + --hash=sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791 \ + --hash=sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73 \ + --hash=sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec \ + --hash=sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4 \ + --hash=sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5 \ + --hash=sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac \ + --hash=sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084 \ + --hash=sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c \ + --hash=sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87 \ + --hash=sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35 \ + --hash=sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250 \ + --hash=sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c \ + --hash=sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826 \ + --hash=sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9 \ + --hash=sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713 \ + --hash=sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1 \ + --hash=sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523 \ + --hash=sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3 \ + --hash=sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78 \ + --hash=sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53 \ + --hash=sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c \ + --hash=sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21 \ + --hash=sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5 \ + --hash=sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff \ + --hash=sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45 \ + --hash=sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110 \ + --hash=sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493 \ + --hash=sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b \ + --hash=sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450 \ + --hash=sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86 \ + --hash=sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8 \ + --hash=sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98 \ + --hash=sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89 \ + --hash=sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66 \ + --hash=sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b \ + --hash=sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8 \ + --hash=sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29 \ + --hash=sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6 \ + --hash=sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc \ + --hash=sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2 \ + --hash=sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788 \ + --hash=sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa \ + --hash=sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151 \ + --hash=sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838 \ + --hash=sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b \ + --hash=sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a \ + --hash=sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d \ + --hash=sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908 \ + --hash=sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0 \ + --hash=sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b \ + --hash=sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c \ + --hash=sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee # via mlflow -pillow==12.1.1 +pillow==12.1.1 \ + --hash=sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9 \ + --hash=sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da \ + --hash=sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f \ + --hash=sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642 \ + --hash=sha256:178aa072084bd88ec759052feca8e56cbb14a60b39322b99a049e58090479713 \ + --hash=sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850 \ + --hash=sha256:1a9b0ee305220b392e1124a764ee4265bd063e54a751a6b62eff69992f457fa9 \ + --hash=sha256:1f1625b72740fdda5d77b4def688eb8fd6490975d06b909fd19f13f391e077e0 \ + --hash=sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9 \ + --hash=sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8 \ + --hash=sha256:21329ec8c96c6e979cd0dfd29406c40c1d52521a90544463057d2aaa937d66a6 \ + --hash=sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd \ + --hash=sha256:2c1fc0f2ca5f96a3c8407e41cca26a16e46b21060fe6d5b099d2cb01412222f5 \ + --hash=sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c \ + --hash=sha256:339ffdcb7cbeaa08221cd401d517d4b1fe7a9ed5d400e4a8039719238620ca35 \ + --hash=sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1 \ + --hash=sha256:36341d06738a9f66c8287cf8b876d24b18db9bd8740fa0672c74e259ad408cff \ + --hash=sha256:365b10bb9417dd4498c0e3b128018c4a624dc11c7b97d8cc54effe3b096f4c38 \ + --hash=sha256:3a5cbdcddad0af3da87cb16b60d23648bc3b51967eb07223e9fed77a82b457c4 \ + --hash=sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af \ + --hash=sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60 \ + --hash=sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986 \ + --hash=sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13 \ + --hash=sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717 \ + --hash=sha256:495c302af3aad1ca67420ddd5c7bd480c8867ad173528767d906428057a11f0e \ + --hash=sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b \ + --hash=sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15 \ + --hash=sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a \ + --hash=sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb \ + --hash=sha256:578510d88c6229d735855e1f278aa305270438d36a05031dfaae5067cc8eb04d \ + --hash=sha256:597bd9c8419bc7c6af5604e55847789b69123bbe25d65cc6ad3012b4f3c98d8b \ + --hash=sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e \ + --hash=sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a \ + --hash=sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f \ + --hash=sha256:5d1f9575a12bed9e9eedd9a4972834b08c97a352bd17955ccdebfeca5913fa0a \ + --hash=sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce \ + --hash=sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc \ + --hash=sha256:600fd103672b925fe62ed08e0d874ea34d692474df6f4bf7ebe148b30f89f39f \ + --hash=sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586 \ + --hash=sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f \ + --hash=sha256:665e1b916b043cef294bc54d47bf02d87e13f769bc4bc5fa225a24b3a6c5aca9 \ + --hash=sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8 \ + --hash=sha256:6c52f062424c523d6c4db85518774cc3d50f5539dd6eed32b8f6229b26f24d40 \ + --hash=sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60 \ + --hash=sha256:7311c0a0dcadb89b36b7025dfd8326ecfa36964e29913074d47382706e516a7c \ + --hash=sha256:7aac39bcf8d4770d089588a2e1dd111cbaa42df5a94be3114222057d68336bd0 \ + --hash=sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334 \ + --hash=sha256:7e7976bf1910a8116b523b9f9f58bf410f3e8aa330cd9a2bb2953f9266ab49af \ + --hash=sha256:8089c852a56c2966cf18835db62d9b34fef7ba74c726ad943928d494fa7f4735 \ + --hash=sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524 \ + --hash=sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf \ + --hash=sha256:89c7e895002bbe49cdc5426150377cbbc04767d7547ed145473f496dfa40408b \ + --hash=sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2 \ + --hash=sha256:8fd420ef0c52c88b5a035a0886f367748c72147b2b8f384c9d12656678dfdfa9 \ + --hash=sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7 \ + --hash=sha256:99c1506ea77c11531d75e3a412832a13a71c7ebc8192ab9e4b2e355555920e3e \ + --hash=sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4 \ + --hash=sha256:9f51079765661884a486727f0729d29054242f74b46186026582b4e4769918e4 \ + --hash=sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b \ + --hash=sha256:a0b1cd6232e2b618adcc54d9882e4e662a089d5768cd188f7c245b4c8c44a397 \ + --hash=sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c \ + --hash=sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e \ + --hash=sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029 \ + --hash=sha256:ab174cd7d29a62dd139c44bf74b698039328f45cb03b4596c43473a46656b2f3 \ + --hash=sha256:ab323b787d6e18b3d91a72fc99b1a2c28651e4358749842b8f8dfacd28ef2052 \ + --hash=sha256:adebb5bee0f0af4909c30db0d890c773d1a92ffe83da908e2e9e720f8edf3984 \ + --hash=sha256:aee2810642b2898bb187ced9b349e95d2a7272930796e022efaf12e99dccd293 \ + --hash=sha256:af9a332e572978f0218686636610555ae3defd1633597be015ed50289a03c523 \ + --hash=sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f \ + --hash=sha256:b66e95d05ba806247aaa1561f080abc7975daf715c30780ff92a20e4ec546e1b \ + --hash=sha256:b81b5e3511211631b3f672a595e3221252c90af017e399056d0faabb9538aa80 \ + --hash=sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f \ + --hash=sha256:bb66b7cc26f50977108790e2456b7921e773f23db5630261102233eb355a3b79 \ + --hash=sha256:c6008de247150668a705a6338156efb92334113421ceecf7438a12c9a12dab23 \ + --hash=sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8 \ + --hash=sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e \ + --hash=sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3 \ + --hash=sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e \ + --hash=sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36 \ + --hash=sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f \ + --hash=sha256:d4ce8e329c93845720cd2014659ca67eac35f6433fd3050393d85f3ecef0dad5 \ + --hash=sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f \ + --hash=sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6 \ + --hash=sha256:e879bb6cd5c73848ef3b2b48b8af9ff08c5b71ecda8048b7dd22d8a33f60be32 \ + --hash=sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20 \ + --hash=sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202 \ + --hash=sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0 \ + --hash=sha256:f975aa7ef9684ce7e2c18a3aa8f8e2106ce1e46b94ab713d156b2898811651d3 \ + --hash=sha256:fbfa2a7c10cc2623f412753cddf391c7f971c52ca40a3f65dc5039b2939e8563 \ + --hash=sha256:fc354a04072b765eccf2204f588a7a532c9511e8b9c7f900e1b64e3e33487090 \ + --hash=sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289 # via matplotlib -prettytable==3.17.0 +prettytable==3.17.0 \ + --hash=sha256:59f2590776527f3c9e8cf9fe7b66dd215837cca96a9c39567414cbc632e8ddb0 \ + --hash=sha256:aad69b294ddbe3e1f95ef8886a060ed1666a0b83018bbf56295f6f226c43d287 # via skops -propcache==0.4.1 +propcache==0.4.1 \ + --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e \ + --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 \ + --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be \ + --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 \ + --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 \ + --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b \ + --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 \ + --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf \ + --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 \ + --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 \ + --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 \ + --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 \ + --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 \ + --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 \ + --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 \ + --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 \ + --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 \ + --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc \ + --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe \ + --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb \ + --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 \ + --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 \ + --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e \ + --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff \ + --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 \ + --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 \ + --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 \ + --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 \ + --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf \ + --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 \ + --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a \ + --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc \ + --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a \ + --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 \ + --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 \ + --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 \ + --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 \ + --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 \ + --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 \ + --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af \ + --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa \ + --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 \ + --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc \ + --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 \ + --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 \ + --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf \ + --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 \ + --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 \ + --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c \ + --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 \ + --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e \ + --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 \ + --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 \ + --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 \ + --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 \ + --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db \ + --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f \ + --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d \ + --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 \ + --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e \ + --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 \ + --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 \ + --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a \ + --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b \ + --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f \ + --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f \ + --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 \ + --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 \ + --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 \ + --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 \ + --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db \ + --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded \ + --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb \ + --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 \ + --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 \ + --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 \ + --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 \ + --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 \ + --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f \ + --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b \ + --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 \ + --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 \ + --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 \ + --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c \ + --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 \ + --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 \ + --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 \ + --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 \ + --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 \ + --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 \ + --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db \ + --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac \ + --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 \ + --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 \ + --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f \ + --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 \ + --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 \ + --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 \ + --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 \ + --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a \ + --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f \ + --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 \ + --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 \ + --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 \ + --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 \ + --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 \ + --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 \ + --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 \ + --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 \ + --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 \ + --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e \ + --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 \ + --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d \ + --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 \ + --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c \ + --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e \ + --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 \ + --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af \ + --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f \ + --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 \ + --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 \ + --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 # via # aiohttp # yarl -protobuf==6.33.5 +protobuf==6.33.5 \ + --hash=sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c \ + --hash=sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02 \ + --hash=sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c \ + --hash=sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd \ + --hash=sha256:8f04fa32763dcdb4973d537d6b54e615cc61108c7cb38fe59310c3192d29510a \ + --hash=sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190 \ + --hash=sha256:a3157e62729aafb8df6da2c03aa5c0937c7266c626ce11a278b6eb7963c4e37c \ + --hash=sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 \ + --hash=sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 \ + --hash=sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b # via # databricks-sdk # googleapis-common-protos # mlflow-skinny # mlflow-tracing # opentelemetry-proto -pyarrow==23.0.1 +pyarrow==23.0.1 \ + --hash=sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07 \ + --hash=sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0 \ + --hash=sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350 \ + --hash=sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb \ + --hash=sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d \ + --hash=sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9 \ + --hash=sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1 \ + --hash=sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500 \ + --hash=sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5 \ + --hash=sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701 \ + --hash=sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c \ + --hash=sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56 \ + --hash=sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7 \ + --hash=sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1 \ + --hash=sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce \ + --hash=sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730 \ + --hash=sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d \ + --hash=sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2 \ + --hash=sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca \ + --hash=sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f \ + --hash=sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8 \ + --hash=sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb \ + --hash=sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125 \ + --hash=sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677 \ + --hash=sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f \ + --hash=sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7 \ + --hash=sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05 \ + --hash=sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9 \ + --hash=sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f \ + --hash=sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2 \ + --hash=sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37 \ + --hash=sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690 \ + --hash=sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8 \ + --hash=sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814 \ + --hash=sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019 \ + --hash=sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67 \ + --hash=sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83 \ + --hash=sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886 \ + --hash=sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2 \ + --hash=sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41 \ + --hash=sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a \ + --hash=sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258 \ + --hash=sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78 \ + --hash=sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5 \ + --hash=sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d \ + --hash=sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222 \ + --hash=sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919 \ + --hash=sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f \ + --hash=sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1 \ + --hash=sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd # via mlflow -pyasn1==0.6.2 +pyasn1==0.6.2 \ + --hash=sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf \ + --hash=sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b # via # pyasn1-modules # rsa -pyasn1-modules==0.4.2 +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 # via google-auth -pycparser==3.0 +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 # via cffi -pydantic==2.12.5 +pydantic==2.12.5 \ + --hash=sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49 \ + --hash=sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d # via # fastapi # litellm @@ -302,116 +2183,839 @@ pydantic==2.12.5 # mlflow-tracing # openai # pydantic-settings -pydantic-core==2.41.5 +pydantic-core==2.41.5 \ + --hash=sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90 \ + --hash=sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740 \ + --hash=sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504 \ + --hash=sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84 \ + --hash=sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33 \ + --hash=sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c \ + --hash=sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0 \ + --hash=sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e \ + --hash=sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0 \ + --hash=sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a \ + --hash=sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34 \ + --hash=sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2 \ + --hash=sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3 \ + --hash=sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815 \ + --hash=sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14 \ + --hash=sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba \ + --hash=sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375 \ + --hash=sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf \ + --hash=sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963 \ + --hash=sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1 \ + --hash=sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808 \ + --hash=sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553 \ + --hash=sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1 \ + --hash=sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2 \ + --hash=sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5 \ + --hash=sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470 \ + --hash=sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2 \ + --hash=sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b \ + --hash=sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660 \ + --hash=sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c \ + --hash=sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093 \ + --hash=sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5 \ + --hash=sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594 \ + --hash=sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008 \ + --hash=sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a \ + --hash=sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a \ + --hash=sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd \ + --hash=sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284 \ + --hash=sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586 \ + --hash=sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869 \ + --hash=sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294 \ + --hash=sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f \ + --hash=sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66 \ + --hash=sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51 \ + --hash=sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc \ + --hash=sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97 \ + --hash=sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a \ + --hash=sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d \ + --hash=sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9 \ + --hash=sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c \ + --hash=sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07 \ + --hash=sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36 \ + --hash=sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e \ + --hash=sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05 \ + --hash=sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e \ + --hash=sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941 \ + --hash=sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3 \ + --hash=sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612 \ + --hash=sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3 \ + --hash=sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b \ + --hash=sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe \ + --hash=sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146 \ + --hash=sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11 \ + --hash=sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60 \ + --hash=sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd \ + --hash=sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b \ + --hash=sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c \ + --hash=sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a \ + --hash=sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460 \ + --hash=sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1 \ + --hash=sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf \ + --hash=sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf \ + --hash=sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858 \ + --hash=sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2 \ + --hash=sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9 \ + --hash=sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2 \ + --hash=sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3 \ + --hash=sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6 \ + --hash=sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770 \ + --hash=sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d \ + --hash=sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc \ + --hash=sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23 \ + --hash=sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26 \ + --hash=sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa \ + --hash=sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8 \ + --hash=sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d \ + --hash=sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3 \ + --hash=sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d \ + --hash=sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034 \ + --hash=sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9 \ + --hash=sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1 \ + --hash=sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56 \ + --hash=sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b \ + --hash=sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c \ + --hash=sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a \ + --hash=sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e \ + --hash=sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9 \ + --hash=sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5 \ + --hash=sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a \ + --hash=sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556 \ + --hash=sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e \ + --hash=sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49 \ + --hash=sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2 \ + --hash=sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9 \ + --hash=sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b \ + --hash=sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc \ + --hash=sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb \ + --hash=sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0 \ + --hash=sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8 \ + --hash=sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82 \ + --hash=sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69 \ + --hash=sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b \ + --hash=sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c \ + --hash=sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75 \ + --hash=sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5 \ + --hash=sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f \ + --hash=sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad \ + --hash=sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b \ + --hash=sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7 \ + --hash=sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425 \ + --hash=sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52 # via pydantic -pydantic-settings==2.13.1 +pydantic-settings==2.13.1 \ + --hash=sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025 \ + --hash=sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237 # via mcp -pygments==2.19.2 +pygments==2.19.2 \ + --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 \ + --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b # via rich -pyjwt==2.11.0 +pyjwt==2.11.0 \ + --hash=sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623 \ + --hash=sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469 # via mcp -pyparsing==3.3.2 +pyparsing==3.3.2 \ + --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ + --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc # via matplotlib -python-dateutil==2.9.0.post0 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via # botocore # graphene # matplotlib # pandas -python-dotenv==1.2.2 +python-dotenv==1.2.2 \ + --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a \ + --hash=sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3 # via # litellm # mlflow-skinny # pydantic-settings # uvicorn -python-engineio==4.13.1 +python-engineio==4.13.1 \ + --hash=sha256:0a853fcef52f5b345425d8c2b921ac85023a04dfcf75d7b74696c61e940fd066 \ + --hash=sha256:f32ad10589859c11053ad7d9bb3c9695cdf862113bfb0d20bc4d890198287399 # via python-socketio -python-multipart==0.0.22 +python-multipart==0.0.22 \ + --hash=sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155 \ + --hash=sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58 # via mcp -python-socketio==5.16.1 +python-socketio==5.16.1 \ + --hash=sha256:a3eb1702e92aa2f2b5d3ba00261b61f062cce51f1cfb6900bf3ab4d1934d2d35 \ + --hash=sha256:f863f98eacce81ceea2e742f6388e10ca3cdd0764be21d30d5196470edf5ea89 # via flask-socketio -pytz==2026.1.post1 +pytz==2026.1.post1 \ + --hash=sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1 \ + --hash=sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a # via pandas -pyyaml==6.0.3 +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 # via # huggingface-hub # mlflow-skinny # uvicorn -referencing==0.37.0 +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 \ + --hash=sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8 # via # jsonschema # jsonschema-specifications -regex==2026.2.28 +regex==2026.2.28 \ + --hash=sha256:00945d007fd74a9084d2ab79b695b595c6b7ba3698972fadd43e23230c6979c1 \ + --hash=sha256:00f2b8d9615aa165fdff0a13f1a92049bfad555ee91e20d246a51aa0b556c60a \ + --hash=sha256:01d65fd24206c8e1e97e2e31b286c59009636c022eb5d003f52760b0f42155d4 \ + --hash=sha256:02473c954af35dd2defeb07e44182f5705b30ea3f351a7cbffa9177beb14da5d \ + --hash=sha256:03a83cc26aa2acda6b8b9dfe748cf9e84cbd390c424a1de34fdcef58961a297a \ + --hash=sha256:09500be324f49b470d907b3ef8af9afe857f5cca486f853853f7945ddbf75911 \ + --hash=sha256:0b1d2b07614d95fa2bf8a63fd1e98bd8fa2b4848dc91b1efbc8ba219fdd73952 \ + --hash=sha256:0d25a10811de831c2baa6aef3c0be91622f44dd8d31dd12e69f6398efb15e48b \ + --hash=sha256:0d5bef2031cbf38757a0b0bc4298bb4824b6332d28edc16b39247228fbdbad97 \ + --hash=sha256:10d28e19bd4888e4abf43bd3925f3c134c52fdf7259219003588a42e24c2aa25 \ + --hash=sha256:180e08a435a0319e6a4821c3468da18dc7001987e1c17ae1335488dfe7518dd8 \ + --hash=sha256:195237dc327858a7721bf8b0bbbef797554bc13563c3591e91cd0767bacbe359 \ + --hash=sha256:19a9c9e0a8f24f39d575a6a854d516b48ffe4cbdcb9de55cb0570a032556ecff \ + --hash=sha256:1c2c95e1a2b0f89d01e821ff4de1be4b5d73d1f4b0bf679fa27c1ad8d2327f1a \ + --hash=sha256:1d367257cd86c1cbb97ea94e77b373a0bbc2224976e247f173d19e8f18b4afa7 \ + --hash=sha256:1e496956106fd59ba6322a8ea17141a27c5040e5ee8f9433ae92d4e5204462a0 \ + --hash=sha256:1f8b17be5c27a684ea6759983c13506bd77bfc7c0347dff41b18ce5ddd2ee09a \ + --hash=sha256:2234059cfe33d9813a3677ef7667999caea9eeaa83fef98eb6ce15c6cf9e0215 \ + --hash=sha256:25b6eb660c5cf4b8c3407a1ed462abba26a926cc9965e164268a3267bcc06a43 \ + --hash=sha256:2954379dd20752e82d22accf3ff465311cbb2bac6c1f92c4afd400e1757f7451 \ + --hash=sha256:2afa673660928d0b63d84353c6c08a8a476ddfc4a47e11742949d182e6863ce8 \ + --hash=sha256:2b2b23587b26496ff5fd40df4278becdf386813ec00dc3533fa43a4cf0e2ad3c \ + --hash=sha256:2fb950ac1d88e6b6a9414381f403797b236f9fa17e1eee07683af72b1634207b \ + --hash=sha256:3935174fa4d9f70525a4367aaff3cb8bc0548129d114260c29d9dfa4a5b41692 \ + --hash=sha256:39bb5727650b9a0275c6a6690f9bb3fe693a7e6cc5c3155b1240aedf8926423e \ + --hash=sha256:3b24bd7e9d85dc7c6a8bd2aa14ecd234274a0248335a02adeb25448aecdd420d \ + --hash=sha256:4390c365fd2d45278f45afd4673cb90f7285f5701607e3ad4274df08e36140ae \ + --hash=sha256:481df4623fa4969c8b11f3433ed7d5e3dc9cec0f008356c3212b3933fb77e3d8 \ + --hash=sha256:4f5c0b182ad4269e7381b7c27fdb0408399881f7a92a4624fd5487f2971dfc11 \ + --hash=sha256:50c2fc924749543e0eacc93ada6aeeb3ea5f6715825624baa0dccaec771668ae \ + --hash=sha256:511f7419f7afab475fd4d639d4aedfc54205bcb0800066753ef68a59f0f330b5 \ + --hash=sha256:516604edd17b1c2c3e579cf4e9b25a53bf8fa6e7cedddf1127804d3e0140ca64 \ + --hash=sha256:52b017b35ac2214d0db5f4f90e303634dc44e4aba4bd6235a27f97ecbe5b0472 \ + --hash=sha256:5a932ea8ad5d0430351ff9c76c8db34db0d9f53c1d78f06022a21f4e290c5c18 \ + --hash=sha256:5cdcc17d935c8f9d3f4db5c2ebe2640c332e3822ad5d23c2f8e0228e6947943a \ + --hash=sha256:5d10303dd18cedfd4d095543998404df656088240bcfd3cd20a8f95b861f74bd \ + --hash=sha256:5e68192bb3a1d6fb2836da24aa494e413ea65853a21505e142e5b1064a595f3d \ + --hash=sha256:64e7c6ad614573e0640f271e811a408d79a9e1fe62a46adb602f598df42a818d \ + --hash=sha256:6591f281cb44dc13de9585b552cec6fc6cf47fb2fe7a48892295ee9bc4a612f9 \ + --hash=sha256:69fc560ccbf08a09dc9b52ab69cacfae51e0ed80dc5693078bdc97db2f91ae96 \ + --hash=sha256:6d63a07e5ec8ce7184452cb00c41c37b49e67dc4f73b2955b5b8e782ea970784 \ + --hash=sha256:6db7bfae0f8a2793ff1f7021468ea55e2699d0790eb58ee6ab36ae43aa00bc5b \ + --hash=sha256:71a911098be38c859ceb3f9a9ce43f4ed9f4c6720ad8684a066ea246b76ad9ff \ + --hash=sha256:73cdcdbba8028167ea81490c7f45280113e41db2c7afb65a276f4711fa3bcbff \ + --hash=sha256:78454178c7df31372ea737996fb7f36b3c2c92cccc641d251e072478afb4babc \ + --hash=sha256:7900157786428a79615a8264dac1f12c9b02957c473c8110c6b1f972dcecaddf \ + --hash=sha256:7ab218076eb0944549e7fe74cf0e2b83a82edb27e81cc87411f76240865e04d5 \ + --hash=sha256:7c1b34dfa72f826f535b20712afa9bb3ba580020e834f3c69866c5bddbf10098 \ + --hash=sha256:851fa70df44325e1e4cdb79c5e676e91a78147b1b543db2aec8734d2add30ec2 \ + --hash=sha256:864cdd1a2ef5716b0ab468af40139e62ede1b3a53386b375ec0786bb6783fc05 \ + --hash=sha256:8710d61737b0c0ce6836b1da7109f20d495e49b3809f30e27e9560be67a257bf \ + --hash=sha256:9036b400b20e4858d56d117108d7813ed07bb7803e3eed766675862131135ca6 \ + --hash=sha256:9185cc63359862a6e80fe97f696e04b0ad9a11c4ac0a4a927f979f611bfe3768 \ + --hash=sha256:948c12ef30ecedb128903c2c2678b339746eb7c689c5c21957c4a23950c96d15 \ + --hash=sha256:94d63db12e45a9b9f064bfe4800cefefc7e5f182052e4c1b774d46a40ab1d9bb \ + --hash=sha256:96f6269a2882fbb0ee76967116b83679dc628e68eaea44e90884b8d53d833881 \ + --hash=sha256:97054c55db06ab020342cc0d35d6f62a465fa7662871190175f1ad6c655c028f \ + --hash=sha256:98adf340100cbe6fbaf8e6dc75e28f2c191b1be50ffefe292fb0e6f6eefdb0d8 \ + --hash=sha256:99985a2c277dcb9ccb63f937451af5d65177af1efdeb8173ac55b61095a0a05c \ + --hash=sha256:9b65d33a17101569f86d9c5966a8b1d7fbf8afdda5a8aa219301b0a80f58cf7d \ + --hash=sha256:9dd450db6458387167e033cfa80887a34c99c81d26da1bf8b0b41bf8c9cac88e \ + --hash=sha256:a25c7701e4f7a70021db9aaf4a4a0a67033c6318752146e03d1b94d32006217e \ + --hash=sha256:a448af01e3d8031c89c5d902040b124a5e921a25c4e5e07a861ca591ce429341 \ + --hash=sha256:a5dac14d0872eeb35260a8e30bac07ddf22adc1e3a0635b52b02e180d17c9c7e \ + --hash=sha256:a729e47d418ea11d03469f321aaf67cdee8954cde3ff2cf8403ab87951ad10f2 \ + --hash=sha256:aaffaecffcd2479ce87aa1e74076c221700b7c804e48e98e62500ee748f0f550 \ + --hash=sha256:b059e71ec363968671693a78c5053bd9cb2fe410f9b8e4657e88377ebd603a2e \ + --hash=sha256:b387a0d092dac157fb026d737dde35ff3e49ef27f285343e7c6401851239df27 \ + --hash=sha256:b389c61aa28a79c2e0527ac36da579869c2e235a5b208a12c5b5318cda2501d8 \ + --hash=sha256:b42f7466e32bf15a961cf09f35fa6323cc72e64d3d2c990b10de1274a5da0a59 \ + --hash=sha256:b49eb78048c6354f49e91e4b77da21257fecb92256b6d599ae44403cab30b05b \ + --hash=sha256:b5acd4b6a95f37c3c3828e5d053a7d4edaedb85de551db0153754924cb7c83e3 \ + --hash=sha256:b8b3f1be1738feadc69f62daa250c933e85c6f34fa378f54a7ff43807c1b9117 \ + --hash=sha256:b8cf76f1a29f0e99dcfd7aef1551a9827588aae5a737fe31442021165f1920dc \ + --hash=sha256:ba55c50f408fb5c346a3a02d2ce0ebc839784e24f7c9684fde328ff063c3cdea \ + --hash=sha256:bba2b18d70eeb7b79950f12f633beeecd923f7c9ad6f6bae28e59b4cb3ab046b \ + --hash=sha256:bbb882061f742eb5d46f2f1bd5304055be0a66b783576de3d7eef1bed4778a6e \ + --hash=sha256:bcb399ed84eabf4282587ba151f2732ad8168e66f1d3f85b1d038868fe547703 \ + --hash=sha256:bd477d5f79920338107f04aa645f094032d9e3030cc55be581df3d1ef61aa318 \ + --hash=sha256:bec23c11cbbf09a4df32fe50d57cbdd777bc442269b6e39a1775654f1c95dee2 \ + --hash=sha256:c0b5ccbb8ffb433939d248707d4a8b31993cb76ab1a0187ca886bf50e96df952 \ + --hash=sha256:c15af43c72a7fb0c97cbc66fa36a43546eddc5c06a662b64a0cbf30d6ac40944 \ + --hash=sha256:c7815afb0ca45456613fdaf60ea9c993715511c8d53a83bc468305cbc0ee23c7 \ + --hash=sha256:cb3b1db8ff6c7b8bf838ab05583ea15230cb2f678e569ab0e3a24d1e8320940b \ + --hash=sha256:d0b02e8b7e5874b48ae0f077ecca61c1a6a9f9895e9c6dfb191b55b242862033 \ + --hash=sha256:d6b08a06976ff4fb0d83077022fde3eca06c55432bb997d8c0495b9a4e9872f4 \ + --hash=sha256:d6cfe798d8da41bb1862ed6e0cba14003d387c3c0c4a5d45591076ae9f0ce2f8 \ + --hash=sha256:d8511a01d0e4ee1992eb3ba19e09bc1866fe03f05129c3aec3fdc4cbc77aad3f \ + --hash=sha256:dc8ed8c3f41c27acb83f7b6a9eb727a73fc6663441890c5cb3426a5f6a91ce7d \ + --hash=sha256:dd8847c4978bc3c7e6c826fb745f5570e518b8459ac2892151ce6627c7bc00d5 \ + --hash=sha256:de0cf053139f96219ccfabb4a8dd2d217c8c82cb206c91d9f109f3f552d6b43d \ + --hash=sha256:dee50f1be42222f89767b64b283283ef963189da0dda4a515aa54a5563c62dec \ + --hash=sha256:e1e7b24cb3ae9953a560c563045d1ba56ee4749fbd05cf21ba571069bd7be81b \ + --hash=sha256:e59bc8f30414d283ae8ee1617b13d8112e7135cb92830f0ec3688cb29152585a \ + --hash=sha256:e61eea47230eba62a31f3e8a0e3164d0f37ef9f40529fb2c79361bc6b53d2a92 \ + --hash=sha256:e621fb7c8dc147419b28e1702f58a0177ff8308a76fa295c71f3e7827849f5d9 \ + --hash=sha256:e71dcecaa113eebcc96622c17692672c2d104b1d71ddf7adeda90da7ddeb26fc \ + --hash=sha256:e7ce83654d1ab701cb619285a18a8e5a889c1216d746ddc710c914ca5fd71022 \ + --hash=sha256:e8c8cb2deba42f5ec1ede46374e990f8adc5e6456a57ac1a261b19be6f28e4e6 \ + --hash=sha256:ec0c608b7a7465ffadb344ed7c987ff2f11ee03f6a130b569aa74d8a70e8333c \ + --hash=sha256:ec6f5674c5dc836994f50f1186dd1fafde4be0666aae201ae2fcc3d29d8adf27 \ + --hash=sha256:edb1b1b3a5576c56f08ac46f108c40333f222ebfd5cf63afdfa3aab0791ebe5b \ + --hash=sha256:ef77bdde9c9eba3f7fa5b58084b29bbcc74bcf55fdbeaa67c102a35b5bd7e7cc \ + --hash=sha256:f2791948f7c70bb9335a9102df45e93d428f4b8128020d85920223925d73b9e1 \ + --hash=sha256:f467cb602f03fbd1ab1908f68b53c649ce393fde056628dc8c7e634dab6bfc07 \ + --hash=sha256:f8ed9a5d4612df9d4de15878f0bc6aa7a268afbe5af21a3fdd97fa19516e978c \ + --hash=sha256:fa539be029844c0ce1114762d2952ab6cfdd7c7c9bd72e0db26b94c3c36dcc5a \ + --hash=sha256:fb1c4ff62277d87a7335f2c1ea4e0387b8f2b3ad88a64efd9943906aafad4f33 \ + --hash=sha256:fb4db2f17e6484904f986c5a657cec85574c76b5c5e61c7aae9ffa1bc6224f95 \ + --hash=sha256:fb66e5245db9652abd7196ace599b04d9c0e4aa7c8f0e2803938377835780081 \ + --hash=sha256:fc48c500838be6882b32748f60a15229d2dea96e59ef341eaa96ec83538f498d \ + --hash=sha256:fcf26c3c6d0da98fada8ae4ef0aa1c3405a431c0a77eb17306d38a89b02adcd7 \ + --hash=sha256:fd0ce43e71d825b7c0661f9c54d4d74bd97c56c3fd102a8985bcfea48236bacb \ + --hash=sha256:fd63453f10d29097cc3dc62d070746523973fb5aa1c66d25f8558bebd47fed61 # via tiktoken -requests==2.32.5 +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf # via # databricks-sdk # docker # mlflow-skinny # tiktoken -rich==14.3.3 +rich==14.3.3 \ + --hash=sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d \ + --hash=sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b # via typer -rpds-py==0.30.0 +rpds-py==0.30.0 \ + --hash=sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f \ + --hash=sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136 \ + --hash=sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3 \ + --hash=sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7 \ + --hash=sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65 \ + --hash=sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4 \ + --hash=sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169 \ + --hash=sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf \ + --hash=sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4 \ + --hash=sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2 \ + --hash=sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c \ + --hash=sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4 \ + --hash=sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3 \ + --hash=sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6 \ + --hash=sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7 \ + --hash=sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89 \ + --hash=sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85 \ + --hash=sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6 \ + --hash=sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa \ + --hash=sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb \ + --hash=sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6 \ + --hash=sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87 \ + --hash=sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856 \ + --hash=sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4 \ + --hash=sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f \ + --hash=sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53 \ + --hash=sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229 \ + --hash=sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad \ + --hash=sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23 \ + --hash=sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db \ + --hash=sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038 \ + --hash=sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27 \ + --hash=sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00 \ + --hash=sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18 \ + --hash=sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083 \ + --hash=sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c \ + --hash=sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738 \ + --hash=sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898 \ + --hash=sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e \ + --hash=sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7 \ + --hash=sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08 \ + --hash=sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6 \ + --hash=sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551 \ + --hash=sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e \ + --hash=sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288 \ + --hash=sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df \ + --hash=sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0 \ + --hash=sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2 \ + --hash=sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05 \ + --hash=sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0 \ + --hash=sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464 \ + --hash=sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5 \ + --hash=sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404 \ + --hash=sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7 \ + --hash=sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139 \ + --hash=sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394 \ + --hash=sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb \ + --hash=sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15 \ + --hash=sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff \ + --hash=sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed \ + --hash=sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6 \ + --hash=sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e \ + --hash=sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95 \ + --hash=sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d \ + --hash=sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950 \ + --hash=sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3 \ + --hash=sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5 \ + --hash=sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97 \ + --hash=sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e \ + --hash=sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e \ + --hash=sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b \ + --hash=sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd \ + --hash=sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad \ + --hash=sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8 \ + --hash=sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425 \ + --hash=sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221 \ + --hash=sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d \ + --hash=sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825 \ + --hash=sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51 \ + --hash=sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e \ + --hash=sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f \ + --hash=sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8 \ + --hash=sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f \ + --hash=sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d \ + --hash=sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07 \ + --hash=sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877 \ + --hash=sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31 \ + --hash=sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58 \ + --hash=sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94 \ + --hash=sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28 \ + --hash=sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000 \ + --hash=sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1 \ + --hash=sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1 \ + --hash=sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7 \ + --hash=sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7 \ + --hash=sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40 \ + --hash=sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d \ + --hash=sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0 \ + --hash=sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84 \ + --hash=sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f \ + --hash=sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a \ + --hash=sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7 \ + --hash=sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419 \ + --hash=sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8 \ + --hash=sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a \ + --hash=sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9 \ + --hash=sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be \ + --hash=sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed \ + --hash=sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a \ + --hash=sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d \ + --hash=sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324 \ + --hash=sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f \ + --hash=sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2 \ + --hash=sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f \ + --hash=sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5 # via # jsonschema # referencing -rsa==4.9.1 +rsa==4.9.1 \ + --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ + --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 # via google-auth -s3transfer==0.16.0 +s3transfer==0.16.0 \ + --hash=sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe \ + --hash=sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920 # via boto3 -scikit-learn==1.8.0 +scikit-learn==1.8.0 \ + --hash=sha256:00d6f1d66fbcf4eba6e356e1420d33cc06c70a45bb1363cd6f6a8e4ebbbdece2 \ + --hash=sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a \ + --hash=sha256:146b4d36f800c013d267b29168813f7a03a43ecd2895d04861f1240b564421da \ + --hash=sha256:15fc3b5d19cc2be65404786857f2e13c70c83dd4782676dd6814e3b89dc8f5b9 \ + --hash=sha256:2838551e011a64e3053ad7618dda9310175f7515f1742fa2d756f7c874c05961 \ + --hash=sha256:29ffc74089f3d5e87dfca4c2c8450f88bdc61b0fc6ed5d267f3988f19a1309f6 \ + --hash=sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271 \ + --hash=sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809 \ + --hash=sha256:3bad7565bc9cf37ce19a7c0d107742b320c1285df7aab1a6e2d28780df167242 \ + --hash=sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4 \ + --hash=sha256:4511be56637e46c25721e83d1a9cea9614e7badc7040c4d573d75fbe257d6fd7 \ + --hash=sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76 \ + --hash=sha256:56079a99c20d230e873ea40753102102734c5953366972a71d5cb39a32bc40c6 \ + --hash=sha256:5e30adb87f0cc81c7690a84f7932dd66be5bac57cfe16b91cb9151683a4a2d3b \ + --hash=sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e \ + --hash=sha256:6b595b07a03069a2b1740dc08c2299993850ea81cce4fe19b2421e0c970de6b7 \ + --hash=sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e \ + --hash=sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57 \ + --hash=sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735 \ + --hash=sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb \ + --hash=sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb \ + --hash=sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e \ + --hash=sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd \ + --hash=sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a \ + --hash=sha256:a69525355a641bf8ef136a7fa447672fb54fe8d60cab5538d9eb7c6438543fb9 \ + --hash=sha256:ada8121bcb4dac28d930febc791a69f7cb1673c8495e5eee274190b73a4559c1 \ + --hash=sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde \ + --hash=sha256:c22a2da7a198c28dd1a6e1136f19c830beab7fdca5b3e5c8bba8394f8a5c45b3 \ + --hash=sha256:c2656924ec73e5939c76ac4c8b026fc203b83d8900362eb2599d8aee80e4880f \ + --hash=sha256:c57b1b610bd1f40ba43970e11ce62821c2e6569e4d74023db19c6b26f246cb3b \ + --hash=sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3 \ + --hash=sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e \ + --hash=sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702 \ + --hash=sha256:f28dd15c6bb0b66ba09728cf09fd8736c304be29409bd8445a080c1280619e8c \ + --hash=sha256:f984ca4b14914e6b4094c5d52a32ea16b49832c03bd17a110f004db3c223e8e1 \ + --hash=sha256:fb65db5d7531bccf3a4f6bec3462223bea71384e2cda41da0f10b7c292b9e7c4 \ + --hash=sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd # via # mlflow # skops -scipy==1.17.1 +scipy==1.17.1 \ + --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ + --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ + --hash=sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118 \ + --hash=sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39 \ + --hash=sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e \ + --hash=sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6 \ + --hash=sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec \ + --hash=sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21 \ + --hash=sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1 \ + --hash=sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6 \ + --hash=sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce \ + --hash=sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8 \ + --hash=sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448 \ + --hash=sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19 \ + --hash=sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b \ + --hash=sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87 \ + --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ + --hash=sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9 \ + --hash=sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b \ + --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ + --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ + --hash=sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87 \ + --hash=sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c \ + --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ + --hash=sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad \ + --hash=sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f \ + --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c \ + --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ + --hash=sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd \ + --hash=sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866 \ + --hash=sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d \ + --hash=sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6 \ + --hash=sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb \ + --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ + --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ + --hash=sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca \ + --hash=sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d \ + --hash=sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee \ + --hash=sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4 \ + --hash=sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717 \ + --hash=sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49 \ + --hash=sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2 \ + --hash=sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a \ + --hash=sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350 \ + --hash=sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950 \ + --hash=sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b \ + --hash=sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086 \ + --hash=sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444 \ + --hash=sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068 \ + --hash=sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff \ + --hash=sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a \ + --hash=sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50 \ + --hash=sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696 \ + --hash=sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21 \ + --hash=sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c \ + --hash=sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484 \ + --hash=sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118 \ + --hash=sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3 \ + --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ + --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ + --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 # via # mlflow # scikit-learn # skops -shellingham==1.5.4 +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via typer -simple-websocket==1.1.0 +simple-websocket==1.1.0 \ + --hash=sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c \ + --hash=sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4 # via # -r requirements.txt # python-engineio -six==1.17.0 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -skops==0.13.0 +skops==0.13.0 \ + --hash=sha256:55e2cccb18c86f5916e4cfe5acf55ed7b0eecddf08a151906414c092fa5926dc \ + --hash=sha256:66949fd3c95cbb5c80270fbe40293c0fe1e46cb4a921860e42584dd9c20ebeb1 # via mlflow -slowapi==0.1.9 +slowapi==0.1.9 \ + --hash=sha256:639192d0f1ca01b1c6d95bf6c71d794c3a9ee189855337b4821f7f457dddad77 \ + --hash=sha256:cfad116cfb84ad9d763ee155c1e5c5cbf00b0d47399a769b227865f5df576e36 # via mlflow -smmap==5.0.3 +smmap==5.0.3 \ + --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \ + --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f # via gitdb -sniffio==1.3.1 +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via openai -sqlalchemy==2.0.48 +sqlalchemy==2.0.48 \ + --hash=sha256:01f6bbd4308b23240cf7d3ef117557c8fd097ec9549d5d8a52977544e35b40ad \ + --hash=sha256:07edba08061bc277bfdc772dd2a1a43978f5a45994dd3ede26391b405c15221e \ + --hash=sha256:10853a53a4a00417a00913d270dddda75815fcb80675874285f41051c094d7dd \ + --hash=sha256:1182437cb2d97988cfea04cf6cdc0b0bb9c74f4d56ec3d08b81e23d621a28cc6 \ + --hash=sha256:144921da96c08feb9e2b052c5c5c1d0d151a292c6135623c6b2c041f2a45f9e0 \ + --hash=sha256:1a89ce07ad2d4b8cfc30bd5889ec40613e028ed80ef47da7d9dd2ce969ad30e0 \ + --hash=sha256:1b4c575df7368b3b13e0cebf01d4679f9a28ed2ae6c1cd0b1d5beffb6b2007dc \ + --hash=sha256:1ccd42229aaac2df431562117ac7e667d702e8e44afdb6cf0e50fa3f18160f0b \ + --hash=sha256:2645b7d8a738763b664a12a1542c89c940daa55196e8d73e55b169cc5c99f65f \ + --hash=sha256:288937433bd44e3990e7da2402fabc44a3c6c25d3704da066b85b89a85474ae0 \ + --hash=sha256:34634e196f620c7a61d18d5cf7dc841ca6daa7961aed75d532b7e58b309ac894 \ + --hash=sha256:348174f228b99f33ca1f773e85510e08927620caa59ffe7803b37170df30332b \ + --hash=sha256:36ac4ddc3d33e852da9cb00ffb08cea62ca05c39711dc67062ca2bb1fae35fd8 \ + --hash=sha256:3713e21ea67bca727eecd4a24bf68bcd414c403faae4989442be60994301ded0 \ + --hash=sha256:389b984139278f97757ea9b08993e7b9d1142912e046ab7d82b3fbaeb0209131 \ + --hash=sha256:426c5ca86415d9b8945c7073597e10de9644802e2ff502b8e1f11a7a2642856b \ + --hash=sha256:4599a95f9430ae0de82b52ff0d27304fe898c17cb5f4099f7438a51b9998ac77 \ + --hash=sha256:49b7bddc1eebf011ea5ab722fdbe67a401caa34a350d278cc7733c0e88fecb1f \ + --hash=sha256:53667b5f668991e279d21f94ccfa6e45b4e3f4500e7591ae59a8012d0f010dcb \ + --hash=sha256:546572a1793cc35857a2ffa1fe0e58571af1779bcc1ffa7c9fb0839885ed69a9 \ + --hash=sha256:583849c743e0e3c9bb7446f5b5addeacedc168d657a69b418063dfdb2d90081c \ + --hash=sha256:5aee45fd2c6c0f2b9cdddf48c48535e7471e42d6fb81adfde801da0bd5b93241 \ + --hash=sha256:5b193a7e29fd9fa56e502920dca47dffe60f97c863494946bd698c6058a55658 \ + --hash=sha256:5ca74f37f3369b45e1f6b7b06afb182af1fd5dde009e4ffd831830d98cbe5fe7 \ + --hash=sha256:68549c403f79a8e25984376480959975212a670405e3913830614432b5daa07a \ + --hash=sha256:69f5bc24904d3bc3640961cddd2523e361257ef68585d6e364166dfbe8c78fae \ + --hash=sha256:6bb85c546591569558571aa1b06aba711b26ae62f111e15e56136d69920e1616 \ + --hash=sha256:6f7b7243850edd0b8b97043f04748f31de50cf426e939def5c16bedb540698f7 \ + --hash=sha256:7001dc9d5f6bb4deb756d5928eaefe1930f6f4179da3924cbd95ee0e9f4dce89 \ + --hash=sha256:7a936f1bb23d370b7c8cc079d5fce4c7d18da87a33c6744e51a93b0f9e97e9b3 \ + --hash=sha256:7c998f2ace8bf76b453b75dbcca500d4f4b9dd3908c13e89b86289b37784848b \ + --hash=sha256:7cddca31edf8b0653090cbb54562ca027c421c58ddde2c0685f49ff56a1690e0 \ + --hash=sha256:8183dc57ae7d9edc1346e007e840a9f3d6aa7b7f165203a99e16f447150140d2 \ + --hash=sha256:82745b03b4043e04600a6b665cb98697c4339b24e34d74b0a2ac0a2488b6f94d \ + --hash=sha256:841a94c66577661c1f088ac958cd767d7c9bf507698f45afffe7a4017049de76 \ + --hash=sha256:858e433f12b0e5b3ed2f8da917433b634f4937d0e8793e5cb33c54a1a01df565 \ + --hash=sha256:908a3fa6908716f803b86896a09a2c4dde5f5ce2bb07aacc71ffebb57986ce99 \ + --hash=sha256:9764014ef5e58aab76220c5664abb5d47d5bc858d9debf821e55cfdd0f128485 \ + --hash=sha256:9c7d0a77e36b5f4b01ca398482230ab792061d243d715299b44a0b55c89fe617 \ + --hash=sha256:a5b429eb84339f9f05e06083f119ad814e6d85e27ecbdf9c551dfdbb128eaf8a \ + --hash=sha256:a66fe406437dd65cacd96a72689a3aaaecaebbcd62d81c5ac1c0fdbeac835096 \ + --hash=sha256:a6b764fb312bd35e47797ad2e63f0d323792837a6ac785a4ca967019357d2bc7 \ + --hash=sha256:b19151e76620a412c2ac1c6f977ab1b9fa7ad43140178345136456d5265b32ed \ + --hash=sha256:b8438ec5594980d405251451c5b7ea9aa58dda38eb7ac35fb7e4c696712ee24f \ + --hash=sha256:b8fc3454b4f3bd0a368001d0e968852dad45a873f8b4babd41bc302ec851a099 \ + --hash=sha256:bcb8ebbf2e2c36cfe01a94f2438012c6a9d494cf80f129d9753bcdf33bfc35a6 \ + --hash=sha256:d404dc897ce10e565d647795861762aa2d06ca3f4a728c5e9a835096c7059018 \ + --hash=sha256:d612c976cbc2d17edfcc4c006874b764e85e990c29ce9bd411f926bbfb02b9a2 \ + --hash=sha256:d64177f443594c8697369c10e4bbcac70ef558e0f7921a1de7e4a3d1734bcf67 \ + --hash=sha256:d854b3970067297f3a7fbd7a4683587134aa9b3877ee15aa29eea478dc68f933 \ + --hash=sha256:d8fcccbbc0c13c13702c471da398b8cd72ba740dca5859f148ae8e0e8e0d3e7e \ + --hash=sha256:e004aa9248e8cb0a5f9b96d003ca7c1c0a5da8decd1066e7b53f59eb8ce7c62b \ + --hash=sha256:e214d546c8ecb5fc22d6e6011746082abf13a9cf46eefb45769c7b31407c97b5 \ + --hash=sha256:e2d0d88686e3d35a76f3e15a34e8c12d73fc94c1dea1cd55782e695cc14086dd \ + --hash=sha256:e2f35b4cccd9ed286ad62e0a3c3ac21e06c02abc60e20aa51a3e305a30f5fa79 \ + --hash=sha256:e3070c03701037aa418b55d36532ecb8f8446ed0135acb71c678dbdf12f5b6e4 \ + --hash=sha256:e5e088bf43f6ee6fec7dbf1ef7ff7774a616c236b5c0cb3e00662dd71a56b571 \ + --hash=sha256:e83e3f959aaa1c9df95c22c528096d94848a1bc819f5d0ebf7ee3df0ca63db6c \ + --hash=sha256:f0dcbc588cd5b725162c076eb9119342f6579c7f7f55057bb7e3c6ff27e13121 \ + --hash=sha256:f27f9da0a7d22b9f981108fd4b62f8b5743423388915a563e651c20d06c1f457 \ + --hash=sha256:f8649a14caa5f8a243628b1d61cf530ad9ae4578814ba726816adb1121fc493e \ + --hash=sha256:fac0fa4e4f55f118fd87177dacb1c6522fe39c28d498d259014020fec9164c29 \ + --hash=sha256:fd08b90d211c086181caed76931ecfa2bdfc83eea3cfccdb0f82abc6c4b876cb # via # alembic # mlflow -sqlparse==0.5.5 +sqlparse==0.5.5 \ + --hash=sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba \ + --hash=sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e # via mlflow-skinny -sse-starlette==3.3.2 +sse-starlette==3.3.2 \ + --hash=sha256:5c3ea3dad425c601236726af2f27689b74494643f57017cafcb6f8c9acfbb862 \ + --hash=sha256:678fca55a1945c734d8472a6cad186a55ab02840b4f6786f5ee8770970579dcd # via mcp -starlette==0.52.1 +starlette==0.52.1 \ + --hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \ + --hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933 # via # fastapi # mcp # sse-starlette -threadpoolctl==3.6.0 +threadpoolctl==3.6.0 \ + --hash=sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb \ + --hash=sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e # via scikit-learn -tiktoken==0.12.0 +tiktoken==0.12.0 \ + --hash=sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa \ + --hash=sha256:04f0e6a985d95913cabc96a741c5ffec525a2c72e9df086ff17ebe35985c800e \ + --hash=sha256:06a9f4f49884139013b138920a4c393aa6556b2f8f536345f11819389c703ebb \ + --hash=sha256:09eb4eae62ae7e4c62364d9ec3a57c62eea707ac9a2b2c5d6bd05de6724ea179 \ + --hash=sha256:0ee8f9ae00c41770b5f9b0bb1235474768884ae157de3beb5439ca0fd70f3e25 \ + --hash=sha256:15d875454bbaa3728be39880ddd11a5a2a9e548c29418b41e8fd8a767172b5ec \ + --hash=sha256:20cf97135c9a50de0b157879c3c4accbb29116bcf001283d26e073ff3b345946 \ + --hash=sha256:285ba9d73ea0d6171e7f9407039a290ca77efcdb026be7769dccc01d2c8d7fff \ + --hash=sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b \ + --hash=sha256:2cff3688ba3c639ebe816f8d58ffbbb0aa7433e23e08ab1cade5d175fc973fb3 \ + --hash=sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5 \ + --hash=sha256:399c3dd672a6406719d84442299a490420b458c44d3ae65516302a99675888f3 \ + --hash=sha256:3de02f5a491cfd179aec916eddb70331814bd6bf764075d39e21d5862e533970 \ + --hash=sha256:3e68e3e593637b53e56f7237be560f7a394451cb8c11079755e80ae64b9e6def \ + --hash=sha256:47a5bc270b8c3db00bb46ece01ef34ad050e364b51d406b6f9730b64ac28eded \ + --hash=sha256:4a1a4fcd021f022bfc81904a911d3df0f6543b9e7627b51411da75ff2fe7a1be \ + --hash=sha256:4c9614597ac94bb294544345ad8cf30dac2129c05e2db8dc53e082f355857af7 \ + --hash=sha256:508fa71810c0efdcd1b898fda574889ee62852989f7c1667414736bcb2b9a4bd \ + --hash=sha256:54c891b416a0e36b8e2045b12b33dd66fb34a4fe7965565f1b482da50da3e86a \ + --hash=sha256:584c3ad3d0c74f5269906eb8a659c8bfc6144a52895d9261cdaf90a0ae5f4de0 \ + --hash=sha256:5edb8743b88d5be814b1a8a8854494719080c28faaa1ccbef02e87354fe71ef0 \ + --hash=sha256:604831189bd05480f2b885ecd2d1986dc7686f609de48208ebbbddeea071fc0b \ + --hash=sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37 \ + --hash=sha256:6de0da39f605992649b9cfa6f84071e3f9ef2cec458d08c5feb1b6f0ff62e134 \ + --hash=sha256:6e227c7f96925003487c33b1b32265fad2fbcec2b7cf4817afb76d416f40f6bb \ + --hash=sha256:6faa0534e0eefbcafaccb75927a4a380463a2eaa7e26000f0173b920e98b720a \ + --hash=sha256:6fb2995b487c2e31acf0a9e17647e3b242235a20832642bb7a9d1a181c0c1bb1 \ + --hash=sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3 \ + --hash=sha256:82991e04fc860afb933efb63957affc7ad54f83e2216fe7d319007dab1ba5892 \ + --hash=sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3 \ + --hash=sha256:8f317e8530bb3a222547b85a58583238c8f74fd7a7408305f9f63246d1a0958b \ + --hash=sha256:981a81e39812d57031efdc9ec59fa32b2a5a5524d20d4776574c4b4bd2e9014a \ + --hash=sha256:9baf52f84a3f42eef3ff4e754a0db79a13a27921b457ca9832cf944c6be4f8f3 \ + --hash=sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160 \ + --hash=sha256:a1af81a6c44f008cba48494089dd98cccb8b313f55e961a52f5b222d1e507967 \ + --hash=sha256:a90388128df3b3abeb2bfd1895b0681412a8d7dc644142519e6f0a97c2111646 \ + --hash=sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931 \ + --hash=sha256:b4e7ed1c6a7a8a60a3230965bdedba8cc58f68926b835e519341413370e0399a \ + --hash=sha256:b6cfb6d9b7b54d20af21a912bfe63a2727d9cfa8fbda642fd8322c70340aad16 \ + --hash=sha256:b8a0cd0c789a61f31bf44851defbd609e8dd1e2c8589c614cc1060940ef1f697 \ + --hash=sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8 \ + --hash=sha256:c06cf0fcc24c2cb2adb5e185c7082a82cba29c17575e828518c2f11a01f445aa \ + --hash=sha256:c2c714c72bc00a38ca969dae79e8266ddec999c7ceccd603cc4f0d04ccd76365 \ + --hash=sha256:cbb9a3ba275165a2cb0f9a83f5d7025afe6b9d0ab01a22b50f0e74fee2ad253e \ + --hash=sha256:cde24cdb1b8a08368f709124f15b36ab5524aac5fa830cc3fdce9c03d4fb8030 \ + --hash=sha256:d186a5c60c6a0213f04a7a802264083dea1bbde92a2d4c7069e1a56630aef830 \ + --hash=sha256:d51d75a5bffbf26f86554d28e78bfb921eae998edc2675650fd04c7e1f0cdc1e \ + --hash=sha256:d5f89ea5680066b68bcb797ae85219c72916c922ef0fcdd3480c7d2315ffff16 \ + --hash=sha256:da900aa0ad52247d8794e307d6446bd3cdea8e192769b56276695d34d2c9aa88 \ + --hash=sha256:dc2dd125a62cb2b3d858484d6c614d136b5b848976794edfb63688d539b8b93f \ + --hash=sha256:df37684ace87d10895acb44b7f447d4700349b12197a526da0d4a4149fde074c \ + --hash=sha256:dfdfaa5ffff8993a3af94d1125870b1d27aed7cb97aa7eb8c1cefdbc87dbee63 \ + --hash=sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad \ + --hash=sha256:f18f249b041851954217e9fd8e5c00b024ab2315ffda5ed77665a05fa91f42dc \ + --hash=sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71 \ + --hash=sha256:fc530a28591a2d74bce821d10b418b26a094bf33839e69042a6e86ddb7a7fb27 \ + --hash=sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd # via # litellm # mlflow -tokenizers==0.22.2 +tokenizers==0.22.2 \ + --hash=sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113 \ + --hash=sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37 \ + --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \ + --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \ + --hash=sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee \ + --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \ + --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \ + --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \ + --hash=sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012 \ + --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \ + --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \ + --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \ + --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \ + --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \ + --hash=sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195 \ + --hash=sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4 \ + --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \ + --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \ + --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \ + --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \ + --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \ + --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b \ + --hash=sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c \ + --hash=sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5 # via litellm -tqdm==4.67.3 +tqdm==4.67.3 \ + --hash=sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb \ + --hash=sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf # via # huggingface-hub # openai -typer==0.24.1 +typer==0.24.1 \ + --hash=sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e \ + --hash=sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45 # via huggingface-hub -typing-extensions==4.15.0 +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via # aiosignal # alembic @@ -434,44 +3038,497 @@ typing-extensions==4.15.0 # sqlalchemy # starlette # typing-inspection -typing-inspection==0.4.2 +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 # via # fastapi # mcp # pydantic # pydantic-settings -tzdata==2025.3 +tzdata==2025.3 \ + --hash=sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1 \ + --hash=sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7 # via pandas -urllib3==2.6.3 +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # docker # requests -uvicorn==0.41.0 +uvicorn==0.41.0 \ + --hash=sha256:09d11cf7008da33113824ee5a1c6422d89fbc2ff476540d69a34c87fab8b571a \ + --hash=sha256:29e35b1d2c36a04b9e180d4007ede3bcb32a85fbdfd6c6aeb3f26839de088187 # via # mcp # mlflow # mlflow-skinny -uvloop==0.22.1 +uvloop==0.22.1 \ + --hash=sha256:017bd46f9e7b78e81606329d07141d3da446f8798c6baeec124260e22c262772 \ + --hash=sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e \ + --hash=sha256:05e4b5f86e621cf3927631789999e697e58f0d2d32675b67d9ca9eb0bca55743 \ + --hash=sha256:0ae676de143db2b2f60a9696d7eca5bb9d0dd6cc3ac3dad59a8ae7e95f9e1b54 \ + --hash=sha256:1489cf791aa7b6e8c8be1c5a080bae3a672791fcb4e9e12249b05862a2ca9cec \ + --hash=sha256:17d4e97258b0172dfa107b89aa1eeba3016f4b1974ce85ca3ef6a66b35cbf659 \ + --hash=sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8 \ + --hash=sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad \ + --hash=sha256:286322a90bea1f9422a470d5d2ad82d38080be0a29c4dd9b3e6384320a4d11e7 \ + --hash=sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35 \ + --hash=sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289 \ + --hash=sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142 \ + --hash=sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77 \ + --hash=sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733 \ + --hash=sha256:481c990a7abe2c6f4fc3d98781cc9426ebd7f03a9aaa7eb03d3bfc68ac2a46bd \ + --hash=sha256:4a968a72422a097b09042d5fa2c5c590251ad484acf910a651b4b620acd7f193 \ + --hash=sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74 \ + --hash=sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0 \ + --hash=sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6 \ + --hash=sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473 \ + --hash=sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21 \ + --hash=sha256:55502bc2c653ed2e9692e8c55cb95b397d33f9f2911e929dc97c4d6b26d04242 \ + --hash=sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705 \ + --hash=sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702 \ + --hash=sha256:57df59d8b48feb0e613d9b1f5e57b7532e97cbaf0d61f7aa9aa32221e84bc4b6 \ + --hash=sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f \ + --hash=sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e \ + --hash=sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d \ + --hash=sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370 \ + --hash=sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4 \ + --hash=sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792 \ + --hash=sha256:80eee091fe128e425177fbd82f8635769e2f32ec9daf6468286ec57ec0313efa \ + --hash=sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079 \ + --hash=sha256:a592b043a47ad17911add5fbd087c76716d7c9ccc1d64ec9249ceafd735f03c2 \ + --hash=sha256:ac33ed96229b7790eb729702751c0e93ac5bc3bcf52ae9eccbff30da09194b86 \ + --hash=sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6 \ + --hash=sha256:b45649628d816c030dba3c80f8e2689bab1c89518ed10d426036cdc47874dfc4 \ + --hash=sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3 \ + --hash=sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21 \ + --hash=sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c \ + --hash=sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e \ + --hash=sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25 \ + --hash=sha256:c3e5c6727a57cb6558592a95019e504f605d1c54eb86463ee9f7a2dbd411c820 \ + --hash=sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9 \ + --hash=sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88 \ + --hash=sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2 \ + --hash=sha256:ea721dd3203b809039fcc2983f14608dae82b212288b346e0bfe46ec2fab0b7c \ + --hash=sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c \ + --hash=sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42 # via uvicorn -watchfiles==1.1.1 +watchfiles==1.1.1 \ + --hash=sha256:00485f441d183717038ed2e887a7c868154f216877653121068107b227a2f64c \ + --hash=sha256:03fa0f5237118a0c5e496185cafa92878568b652a2e9a9382a5151b1a0380a43 \ + --hash=sha256:04e78dd0b6352db95507fd8cb46f39d185cf8c74e4cf1e4fbad1d3df96faf510 \ + --hash=sha256:059098c3a429f62fc98e8ec62b982230ef2c8df68c79e826e37b895bc359a9c0 \ + --hash=sha256:08af70fd77eee58549cd69c25055dc344f918d992ff626068242259f98d598a2 \ + --hash=sha256:0b495de0bb386df6a12b18335a0285dda90260f51bdb505503c02bcd1ce27a8b \ + --hash=sha256:130e4876309e8686a5e37dba7d5e9bc77e6ed908266996ca26572437a5271e18 \ + --hash=sha256:14e0b1fe858430fc0251737ef3824c54027bedb8c37c38114488b8e131cf8219 \ + --hash=sha256:17ef139237dfced9da49fb7f2232c86ca9421f666d78c264c7ffca6601d154c3 \ + --hash=sha256:1a0bb430adb19ef49389e1ad368450193a90038b5b752f4ac089ec6942c4dff4 \ + --hash=sha256:1db5d7ae38ff20153d542460752ff397fcf5c96090c1230803713cf3147a6803 \ + --hash=sha256:28475ddbde92df1874b6c5c8aaeb24ad5be47a11f87cde5a28ef3835932e3e94 \ + --hash=sha256:2edc3553362b1c38d9f06242416a5d8e9fe235c204a4072e988ce2e5bb1f69f6 \ + --hash=sha256:30f7da3fb3f2844259cba4720c3fc7138eb0f7b659c38f3bfa65084c7fc7abce \ + --hash=sha256:311ff15a0bae3714ffb603e6ba6dbfba4065ab60865d15a6ec544133bdb21099 \ + --hash=sha256:319b27255aacd9923b8a276bb14d21a5f7ff82564c744235fc5eae58d95422ae \ + --hash=sha256:35c53bd62a0b885bf653ebf6b700d1bf05debb78ad9292cf2a942b23513dc4c4 \ + --hash=sha256:36193ed342f5b9842edd3532729a2ad55c4160ffcfa3700e0d54be496b70dd43 \ + --hash=sha256:39574d6370c4579d7f5d0ad940ce5b20db0e4117444e39b6d8f99db5676c52fd \ + --hash=sha256:399600947b170270e80134ac854e21b3ccdefa11a9529a3decc1327088180f10 \ + --hash=sha256:3a476189be23c3686bc2f4321dd501cb329c0a0469e77b7b534ee10129ae6374 \ + --hash=sha256:3ad9fe1dae4ab4212d8c91e80b832425e24f421703b5a42ef2e4a1e215aff051 \ + --hash=sha256:3bc570d6c01c206c46deb6e935a260be44f186a2f05179f52f7fcd2be086a94d \ + --hash=sha256:3dbd8cbadd46984f802f6d479b7e3afa86c42d13e8f0f322d669d79722c8ec34 \ + --hash=sha256:3e6f39af2eab0118338902798b5aa6664f46ff66bc0280de76fca67a7f262a49 \ + --hash=sha256:3f53fa183d53a1d7a8852277c92b967ae99c2d4dcee2bfacff8868e6e30b15f7 \ + --hash=sha256:3f6d37644155fb5beca5378feb8c1708d5783145f2a0f1c4d5a061a210254844 \ + --hash=sha256:3f7eb7da0eb23aa2ba036d4f616d46906013a68caf61b7fdbe42fc8b25132e77 \ + --hash=sha256:3fa0b59c92278b5a7800d3ee7733da9d096d4aabcfabb9a928918bd276ef9b9b \ + --hash=sha256:421e29339983e1bebc281fab40d812742268ad057db4aee8c4d2bce0af43b741 \ + --hash=sha256:4b943d3668d61cfa528eb949577479d3b077fd25fb83c641235437bc0b5bc60e \ + --hash=sha256:526e86aced14a65a5b0ec50827c745597c782ff46b571dbfe46192ab9e0b3c33 \ + --hash=sha256:52e06553899e11e8074503c8e716d574adeeb7e68913115c4b3653c53f9bae42 \ + --hash=sha256:544364b2b51a9b0c7000a4b4b02f90e9423d97fbbf7e06689236443ebcad81ab \ + --hash=sha256:5524298e3827105b61951a29c3512deb9578586abf3a7c5da4a8069df247cccc \ + --hash=sha256:55c7475190662e202c08c6c0f4d9e345a29367438cf8e8037f3155e10a88d5a5 \ + --hash=sha256:563b116874a9a7ce6f96f87cd0b94f7faf92d08d0021e837796f0a14318ef8da \ + --hash=sha256:57ca5281a8b5e27593cb7d82c2ac927ad88a96ed406aa446f6344e4328208e9e \ + --hash=sha256:5c85794a4cfa094714fb9c08d4a218375b2b95b8ed1666e8677c349906246c05 \ + --hash=sha256:5f3bde70f157f84ece3765b42b4a52c6ac1a50334903c6eaf765362f6ccca88a \ + --hash=sha256:5f3f58818dc0b07f7d9aa7fe9eb1037aecb9700e63e1f6acfed13e9fef648f5d \ + --hash=sha256:5fac835b4ab3c6487b5dbad78c4b3724e26bcc468e886f8ba8cc4306f68f6701 \ + --hash=sha256:620bae625f4cb18427b1bb1a2d9426dc0dd5a5ba74c7c2cdb9de405f7b129863 \ + --hash=sha256:672b8adf25b1a0d35c96b5888b7b18699d27d4194bac8beeae75be4b7a3fc9b2 \ + --hash=sha256:6aae418a8b323732fa89721d86f39ec8f092fc2af67f4217a2b07fd3e93c6101 \ + --hash=sha256:6c3631058c37e4a0ec440bf583bc53cdbd13e5661bb6f465bc1d88ee9a0a4d02 \ + --hash=sha256:6c9c9262f454d1c4d8aaa7050121eb4f3aea197360553699520767daebf2180b \ + --hash=sha256:6e43d39a741e972bab5d8100b5cdacf69db64e34eb19b6e9af162bccf63c5cc6 \ + --hash=sha256:7365b92c2e69ee952902e8f70f3ba6360d0d596d9299d55d7d386df84b6941fb \ + --hash=sha256:743185e7372b7bc7c389e1badcc606931a827112fbbd37f14c537320fca08620 \ + --hash=sha256:74472234c8370669850e1c312490f6026d132ca2d396abfad8830b4f1c096957 \ + --hash=sha256:74d5012b7630714b66be7b7b7a78855ef7ad58e8650c73afc4c076a1f480a8d6 \ + --hash=sha256:77a13aea58bc2b90173bc69f2a90de8e282648939a00a602e1dc4ee23e26b66d \ + --hash=sha256:79ff6c6eadf2e3fc0d7786331362e6ef1e51125892c75f1004bd6b52155fb956 \ + --hash=sha256:831a62658609f0e5c64178211c942ace999517f5770fe9436be4c2faeba0c0ef \ + --hash=sha256:836398932192dae4146c8f6f737d74baeac8b70ce14831a239bdb1ca882fc261 \ + --hash=sha256:842178b126593addc05acf6fce960d28bc5fae7afbaa2c6c1b3a7b9460e5be02 \ + --hash=sha256:8526e8f916bb5b9a0a777c8317c23ce65de259422bba5b31325a6fa6029d33af \ + --hash=sha256:859e43a1951717cc8de7f4c77674a6d389b106361585951d9e69572823f311d9 \ + --hash=sha256:88863fbbc1a7312972f1c511f202eb30866370ebb8493aef2812b9ff28156a21 \ + --hash=sha256:89eef07eee5e9d1fda06e38822ad167a044153457e6fd997f8a858ab7564a336 \ + --hash=sha256:8c89f9f2f740a6b7dcc753140dd5e1ab9215966f7a3530d0c0705c83b401bd7d \ + --hash=sha256:8c91ed27800188c2ae96d16e3149f199d62f86c7af5f5f4d2c61a3ed8cd3666c \ + --hash=sha256:8ca65483439f9c791897f7db49202301deb6e15fe9f8fe2fed555bf986d10c31 \ + --hash=sha256:8fbe85cb3201c7d380d3d0b90e63d520f15d6afe217165d7f98c9c649654db81 \ + --hash=sha256:91d4c9a823a8c987cce8fa2690923b069966dabb196dd8d137ea2cede885fde9 \ + --hash=sha256:9bb9f66367023ae783551042d31b1d7fd422e8289eedd91f26754a66f44d5cff \ + --hash=sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2 \ + --hash=sha256:a36d8efe0f290835fd0f33da35042a1bb5dc0e83cbc092dcf69bce442579e88e \ + --hash=sha256:a55f3e9e493158d7bfdb60a1165035f1cf7d320914e7b7ea83fe22c6023b58fc \ + --hash=sha256:a625815d4a2bdca61953dbba5a39d60164451ef34c88d751f6c368c3ea73d404 \ + --hash=sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01 \ + --hash=sha256:ac3cc5759570cd02662b15fbcd9d917f7ecd47efe0d6b40474eafd246f91ea18 \ + --hash=sha256:acb08650863767cbc58bca4813b92df4d6c648459dcaa3d4155681962b2aa2d3 \ + --hash=sha256:aebfd0861a83e6c3d1110b78ad54704486555246e542be3e2bb94195eabb2606 \ + --hash=sha256:afaeff7696e0ad9f02cbb8f56365ff4686ab205fcf9c4c5b6fdfaaa16549dd04 \ + --hash=sha256:b27cf2eb1dda37b2089e3907d8ea92922b673c0c427886d4edc6b94d8dfe5db3 \ + --hash=sha256:b2cd9e04277e756a2e2d2543d65d1e2166d6fd4c9b183f8808634fda23f17b14 \ + --hash=sha256:b9c4702f29ca48e023ffd9b7ff6b822acdf47cb1ff44cb490a3f1d5ec8987e9c \ + --hash=sha256:bbe1ef33d45bc71cf21364df962af171f96ecaeca06bd9e3d0b583efb12aec82 \ + --hash=sha256:bd404be08018c37350f0d6e34676bd1e2889990117a2b90070b3007f172d0610 \ + --hash=sha256:bf0a91bfb5574a2f7fc223cf95eeea79abfefa404bf1ea5e339c0c1560ae99a0 \ + --hash=sha256:bfb5862016acc9b869bb57284e6cb35fdf8e22fe59f7548858e2f971d045f150 \ + --hash=sha256:bfff9740c69c0e4ed32416f013f3c45e2ae42ccedd1167ef2d805c000b6c71a5 \ + --hash=sha256:c1f5210f1b8fc91ead1283c6fd89f70e76fb07283ec738056cf34d51e9c1d62c \ + --hash=sha256:c2047d0b6cea13b3316bdbafbfa0c4228ae593d995030fda39089d36e64fc03a \ + --hash=sha256:c22c776292a23bfc7237a98f791b9ad3144b02116ff10d820829ce62dff46d0b \ + --hash=sha256:c755367e51db90e75b19454b680903631d41f9e3607fbd941d296a020c2d752d \ + --hash=sha256:c882d69f6903ef6092bedfb7be973d9319940d56b8427ab9187d1ecd73438a70 \ + --hash=sha256:cb467c999c2eff23a6417e58d75e5828716f42ed8289fe6b77a7e5a91036ca70 \ + --hash=sha256:cdab464fee731e0884c35ae3588514a9bcf718d0e2c82169c1c4a85cc19c3c7f \ + --hash=sha256:ce19e06cbda693e9e7686358af9cd6f5d61312ab8b00488bc36f5aabbaf77e24 \ + --hash=sha256:ce70f96a46b894b36eba678f153f052967a0d06d5b5a19b336ab0dbbd029f73e \ + --hash=sha256:cf57a27fb986c6243d2ee78392c503826056ffe0287e8794503b10fb51b881be \ + --hash=sha256:d1715143123baeeaeadec0528bb7441103979a1d5f6fd0e1f915383fea7ea6d5 \ + --hash=sha256:d6ff426a7cb54f310d51bfe83fe9f2bbe40d540c741dc974ebc30e6aa238f52e \ + --hash=sha256:d7e7067c98040d646982daa1f37a33d3544138ea155536c2e0e63e07ff8a7e0f \ + --hash=sha256:db476ab59b6765134de1d4fe96a1a9c96ddf091683599be0f26147ea1b2e4b88 \ + --hash=sha256:dcc5c24523771db3a294c77d94771abcfcb82a0e0ee8efd910c37c59ec1b31bb \ + --hash=sha256:de6da501c883f58ad50db3a32ad397b09ad29865b5f26f64c24d3e3281685849 \ + --hash=sha256:e84087b432b6ac94778de547e08611266f1f8ffad28c0ee4c82e028b0fc5966d \ + --hash=sha256:eef58232d32daf2ac67f42dea51a2c80f0d03379075d44a587051e63cc2e368c \ + --hash=sha256:f096076119da54a6080e8920cbdaac3dbee667eb91dcc5e5b78840b87415bd44 \ + --hash=sha256:f0ab1c1af0cb38e3f598244c17919fb1a84d1629cc08355b0074b6d7f53138ac \ + --hash=sha256:f27db948078f3823a6bb3b465180db8ebecf26dd5dae6f6180bd87383b6b4428 \ + --hash=sha256:f537afb3276d12814082a2e9b242bdcf416c2e8fd9f799a737990a1dbe906e5b \ + --hash=sha256:f57b396167a2565a4e8b5e56a5a1c537571733992b226f4f1197d79e94cf0ae5 \ + --hash=sha256:f8979280bdafff686ba5e4d8f97840f929a87ed9cdf133cbbd42f7766774d2aa \ + --hash=sha256:f9a2ae5c91cecc9edd47e041a930490c31c3afb1f5e6d71de3dc671bfaca02bf # via # mlflow # uvicorn -wcwidth==0.6.0 +wcwidth==0.6.0 \ + --hash=sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad \ + --hash=sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159 # via prettytable -websockets==16.0 +websockets==16.0 \ + --hash=sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c \ + --hash=sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a \ + --hash=sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe \ + --hash=sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e \ + --hash=sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec \ + --hash=sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1 \ + --hash=sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64 \ + --hash=sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3 \ + --hash=sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8 \ + --hash=sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206 \ + --hash=sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3 \ + --hash=sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156 \ + --hash=sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d \ + --hash=sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9 \ + --hash=sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad \ + --hash=sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2 \ + --hash=sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03 \ + --hash=sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8 \ + --hash=sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230 \ + --hash=sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8 \ + --hash=sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea \ + --hash=sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641 \ + --hash=sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957 \ + --hash=sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6 \ + --hash=sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6 \ + --hash=sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5 \ + --hash=sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f \ + --hash=sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00 \ + --hash=sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e \ + --hash=sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b \ + --hash=sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72 \ + --hash=sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39 \ + --hash=sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9 \ + --hash=sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79 \ + --hash=sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0 \ + --hash=sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac \ + --hash=sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35 \ + --hash=sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0 \ + --hash=sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5 \ + --hash=sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c \ + --hash=sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8 \ + --hash=sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1 \ + --hash=sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244 \ + --hash=sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3 \ + --hash=sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767 \ + --hash=sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a \ + --hash=sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d \ + --hash=sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd \ + --hash=sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e \ + --hash=sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944 \ + --hash=sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82 \ + --hash=sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d \ + --hash=sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4 \ + --hash=sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5 \ + --hash=sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904 \ + --hash=sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde \ + --hash=sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f \ + --hash=sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c \ + --hash=sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89 \ + --hash=sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da \ + --hash=sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4 # via uvicorn -werkzeug==3.1.6 +werkzeug==3.1.6 \ + --hash=sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25 \ + --hash=sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131 # via # flask # flask-cors # flask-socketio -wrapt==2.1.2 +wrapt==2.1.2 \ + --hash=sha256:08ffa54146a7559f5b8df4b289b46d963a8e74ed16ba3687f99896101a3990c5 \ + --hash=sha256:0fc04bc8664a8bc4c8e00b37b5355cffca2535209fba1abb09ae2b7c76ddf82b \ + --hash=sha256:1370e516598854e5b4366e09ce81e08bfe94d42b0fd569b88ec46cc56d9164a9 \ + --hash=sha256:162e4e2ba7542da9027821cb6e7c5e068d64f9a10b5f15512ea28e954893a267 \ + --hash=sha256:16997dfb9d67addc2e3f41b62a104341e80cac52f91110dece393923c0ebd5ca \ + --hash=sha256:1c51c738d7d9faa0b3601708e7e2eda9bf779e1b601dce6c77411f2a1b324a63 \ + --hash=sha256:1c6cc827c00dc839350155f316f1f8b4b0c370f52b6a19e782e2bda89600c7dc \ + --hash=sha256:2b8b28e97a44d21836259739ae76284e180b18abbb4dcfdff07a415cf1016c3e \ + --hash=sha256:2d3ff4f0024dd224290c0eabf0240f1bfc1f26363431505fb1b0283d3b08f11d \ + --hash=sha256:305d8a1755116bfdad5dda9e771dcb2138990a1d66e9edd81658816edf51aed1 \ + --hash=sha256:3144b027ff30cbd2fca07c0a87e67011adb717eb5f5bd8496325c17e454257a3 \ + --hash=sha256:3278c471f4468ad544a691b31bb856374fbdefb7fee1a152153e64019379f015 \ + --hash=sha256:3756219045f73fb28c5d7662778e4156fbd06cf823c4d2d4b19f97305e52819c \ + --hash=sha256:3769a77df8e756d65fbc050333f423c01ae012b4f6731aaf70cf2bef61b34596 \ + --hash=sha256:3969c56e4563c375861c8df14fa55146e81ac11c8db49ea6fb7f2ba58bc1ff9a \ + --hash=sha256:3996a67eecc2c68fd47b4e3c564405a5777367adfd9b8abb58387b63ee83b21e \ + --hash=sha256:3b8d15e52e195813efe5db8cec156eebe339aaf84222f4f4f051a6c01f237ed7 \ + --hash=sha256:3beb22f674550d5634642c645aba4c72a2c66fb185ae1aebe1e955fae5a13baf \ + --hash=sha256:3d7b6fd105f8b24e5bd23ccf41cb1d1099796524bcc6f7fbb8fe576c44befbc9 \ + --hash=sha256:4006c351de6d5007aa33a551f600404ba44228a89e833d2fadc5caa5de8edfbf \ + --hash=sha256:45914e8efbe4b9d5102fcf0e8e2e3258b83a5d5fba9f8f7b6d15681e9d29ffe0 \ + --hash=sha256:467e7c76315390331c67073073d00662015bb730c566820c9ca9b54e4d67fd04 \ + --hash=sha256:478282ebd3795a089154fb16d3db360e103aa13d3b2ad30f8f6aac0d2207de0e \ + --hash=sha256:4b7a86d99a14f76facb269dc148590c01aaf47584071809a70da30555228158c \ + --hash=sha256:4bdf26e03e6d0da3f0e9422fd36bcebf7bc0eeb55fdf9c727a09abc6b9fe472e \ + --hash=sha256:5681123e60aed0e64c7d44f72bbf8b4ce45f79d81467e2c4c728629f5baf06eb \ + --hash=sha256:577dff354e7acd9d411eaf4bfe76b724c89c89c8fc9b7e127ee28c5f7bcb25b6 \ + --hash=sha256:57d7c0c980abdc5f1d98b11a2aa3bb159790add80258c717fa49a99921456d90 \ + --hash=sha256:5a0a0a3a882393095573344075189eb2d566e0fd205a2b6414e9997b1b800a8b \ + --hash=sha256:5c35b5d82b16a3bc6e0a04349b606a0582bc29f573786aebe98e0c159bc48db6 \ + --hash=sha256:5e0fa9cc32300daf9eb09a1f5bdc6deb9a79defd70d5356ba453bcd50aef3742 \ + --hash=sha256:62503ffbc2d3a69891cf29beeaccdb4d5e0a126e2b6a851688d4777e01428dbb \ + --hash=sha256:6433ea84e1cfacf32021d2a4ee909554ade7fd392caa6f7c13f1f4bf7b8e8748 \ + --hash=sha256:64a07a71d2730ba56f11d1a4b91f7817dc79bc134c11516b75d1921a7c6fcda1 \ + --hash=sha256:6de1a3851c27e0bd6a04ca993ea6f80fc53e6c742ee1601f486c08e9f9b900a9 \ + --hash=sha256:6f2c5390460de57fa9582bc8a1b7a6c86e1a41dfad74c5225fc07044c15cc8d1 \ + --hash=sha256:6f8dbdd3719e534860d6a78526aafc220e0241f981367018c2875178cf83a413 \ + --hash=sha256:6f97edc9842cf215312b75fe737ee7c8adda75a89979f8e11558dfff6343cc4b \ + --hash=sha256:710f6e5dfaf6a5d5c397d2d6758a78fecd9649deb21f1b645f5b57a328d63050 \ + --hash=sha256:72aaa9d0d8e4ed0e2e98019cea47a21f823c9dd4b43c7b77bba6679ffcca6a00 \ + --hash=sha256:76405518ca4e1b76fbb1b9f686cff93aebae03920cc55ceeec48ff9f719c5f67 \ + --hash=sha256:767c0dbbe76cae2a60dd2b235ac0c87c9cccf4898aef8062e57bead46b5f6894 \ + --hash=sha256:776867878e83130c7a04237010463372e877c1c994d449ca6aaafeab6aab2586 \ + --hash=sha256:787fd6f4d67befa6fe2abdffcbd3de2d82dfc6fb8a6d850407c53332709d030b \ + --hash=sha256:79847b83eb38e70d93dc392c7c5b587efe65b3e7afcc167aa8abd5d60e8761c8 \ + --hash=sha256:7dfa9f2cf65d027b951d05c662cc99ee3bd01f6e4691ed39848a7a5fffc902b2 \ + --hash=sha256:84ce8f1c2104d2f6daa912b1b5b039f331febfeee74f8042ad4e04992bd95c8f \ + --hash=sha256:866abdbf4612e0b34764922ef8b1c5668867610a718d3053d59e24a5e5fcfc15 \ + --hash=sha256:96159a0ee2b0277d44201c3b5be479a9979cf154e8c82fa5df49586a8e7679bb \ + --hash=sha256:970d57ed83fa040d8b20c52fe74a6ae7e3775ae8cff5efd6a81e06b19078484c \ + --hash=sha256:98ba61833a77b747901e9012072f038795de7fc77849f1faa965464f3f87ff2d \ + --hash=sha256:9c691a6bc752c0cc4711cc0c00896fcd0f116abc253609ef64ef930032821842 \ + --hash=sha256:a5d516e22aedb7c9c1d47cba1c63160b1a6f61ec2f3948d127cd38d5cfbb556f \ + --hash=sha256:a76d61a2e851996150ba0f80582dd92a870643fa481f3b3846f229de88caf044 \ + --hash=sha256:a819e39017f95bf7aede768f75915635aa8f671f2993c036991b8d3bfe8dbb6f \ + --hash=sha256:a8914c754d3134a3032601c6984db1c576e6abaf3fc68094bb8ab1379d75ff92 \ + --hash=sha256:a9372fc3639a878c8e7d87e1556fa209091b0a66e912c611e3f833e2c4202be2 \ + --hash=sha256:a93cd767e37faeddbe07d8fc4212d5cba660af59bdb0f6372c93faaa13e6e679 \ + --hash=sha256:a9b9d50c9af998875a1482a038eb05755dfd6fe303a313f6a940bb53a83c3f18 \ + --hash=sha256:a9dd9813825f7ecb018c17fd147a01845eb330254dff86d3b5816f20f4d6aaf8 \ + --hash=sha256:b89f095fe98bc12107f82a9f7d570dc83a0870291aeb6b1d7a7d35575f55d98a \ + --hash=sha256:b8aefb4dbb18d904b96827435a763fa42fc1f08ea096a391710407a60983ced8 \ + --hash=sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8 \ + --hash=sha256:bbac24d879aa22998e87f6b3f481a5216311e7d53c7db87f189a7a0266dafffb \ + --hash=sha256:c0be8b5a74c5824e9359b53e7e58bef71a729bacc82e16587db1c4ebc91f7c5a \ + --hash=sha256:c20b757c268d30d6215916a5fa8461048d023865d888e437fab451139cad6c8e \ + --hash=sha256:c7e6cd120ef837d5b6f860a6ea3745f8763805c418bb2f12eeb1fa6e25f22d22 \ + --hash=sha256:c87cf3f0c85e27b3ac7d9ad95da166bf8739ca215a8b171e8404a2d739897a45 \ + --hash=sha256:c8e46ae8e4032792eb2f677dbd0d557170a8e5524d22acc55199f43efedd39bf \ + --hash=sha256:cef91c95a50596fcdc31397eb6955476f82ae8a3f5a8eabdc13611b60ee380ba \ + --hash=sha256:d1c5fea4f9fe3762e2b905fdd67df51e4be7a73b7674957af2d2ade71a5c075d \ + --hash=sha256:d307aa6888d5efab2c1cde09843d48c843990be13069003184b67d426d145394 \ + --hash=sha256:d8f7740e1af13dff2684e4d56fe604a7e04d6c94e737a60568d8d4238b9a0c71 \ + --hash=sha256:da1f00a557c66225d53b095a97eace0fc5349e3bfda28fa34ffae238978ee575 \ + --hash=sha256:dad63212b168de8569b1c512f4eac4b57f2c6934b30df32d6ee9534a79f1493f \ + --hash=sha256:de9f1a2bbc5ac7f6012ec24525bdd444765a2ff64b5985ac6e0692144838542e \ + --hash=sha256:e3d3b35eedcf5f7d022291ecd7533321c4775f7b9cd0050a31a68499ba45757c \ + --hash=sha256:e5aeab8fe15c3dff75cfee94260dcd9cded012d4ff06add036c28fae7718593b \ + --hash=sha256:e6ed62c82ddf58d001096ae84ce7f833db97ae2263bff31c9b336ba8cfe3f508 \ + --hash=sha256:eba8155747eb2cae4a0b913d9ebd12a1db4d860fc4c829d7578c7b989bd3f2f0 \ + --hash=sha256:f01277d9a5fc1862f26f7626da9cf443bebc0abd2f303f41c5e995b15887dabd \ + --hash=sha256:f069e113743a21a3defac6677f000068ebb931639f789b5b226598e247a4c89e \ + --hash=sha256:f0d8fc30a43b5fe191cf2b1a0c82bab2571dadd38e7c0062ee87d6df858dd06e \ + --hash=sha256:f29c827a8d9936ac320746747a016c4bc66ef639f5cd0d32df24f5eacbf9c69f \ + --hash=sha256:f3b7d73012ea75aee5844de58c88f44cf62d0d62711e39da5a82824a7c4626a8 \ + --hash=sha256:f8bc1c264d8d1cf5b3560a87bbdd31131573eb25f9f9447bb6252b8d4c44a3a1 \ + --hash=sha256:f8fba1bae256186a83d1875b2b1f4e2d1242e8fac0f58ec0d7e41b26967b965c \ + --hash=sha256:fab036efe5464ec3291411fabb80a7a39e2dd80bae9bcbeeca5087fdfa891e19 \ + --hash=sha256:ff2aad9c4cda28a8f0653fc2d487596458c2a3f475e56ba02909e950a9efa6a9 \ + --hash=sha256:ff95d4264e55839be37bafe1536db2ab2de19da6b65f9244f01f332b5286cfbf # via deprecated -wsproto==1.3.2 +wsproto==1.3.2 \ + --hash=sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584 \ + --hash=sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294 # via simple-websocket -yarl==1.23.0 +yarl==1.23.0 \ + --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \ + --hash=sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4 \ + --hash=sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85 \ + --hash=sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993 \ + --hash=sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222 \ + --hash=sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de \ + --hash=sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25 \ + --hash=sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e \ + --hash=sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2 \ + --hash=sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e \ + --hash=sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860 \ + --hash=sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957 \ + --hash=sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760 \ + --hash=sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52 \ + --hash=sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788 \ + --hash=sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912 \ + --hash=sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719 \ + --hash=sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035 \ + --hash=sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220 \ + --hash=sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412 \ + --hash=sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05 \ + --hash=sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41 \ + --hash=sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 \ + --hash=sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4 \ + --hash=sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd \ + --hash=sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748 \ + --hash=sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a \ + --hash=sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4 \ + --hash=sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34 \ + --hash=sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069 \ + --hash=sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25 \ + --hash=sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2 \ + --hash=sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb \ + --hash=sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f \ + --hash=sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5 \ + --hash=sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8 \ + --hash=sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c \ + --hash=sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512 \ + --hash=sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6 \ + --hash=sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5 \ + --hash=sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9 \ + --hash=sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072 \ + --hash=sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5 \ + --hash=sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277 \ + --hash=sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a \ + --hash=sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6 \ + --hash=sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae \ + --hash=sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26 \ + --hash=sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2 \ + --hash=sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4 \ + --hash=sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70 \ + --hash=sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723 \ + --hash=sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c \ + --hash=sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9 \ + --hash=sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5 \ + --hash=sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e \ + --hash=sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c \ + --hash=sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4 \ + --hash=sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0 \ + --hash=sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2 \ + --hash=sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b \ + --hash=sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7 \ + --hash=sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750 \ + --hash=sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2 \ + --hash=sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474 \ + --hash=sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716 \ + --hash=sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7 \ + --hash=sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123 \ + --hash=sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007 \ + --hash=sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595 \ + --hash=sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe \ + --hash=sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea \ + --hash=sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598 \ + --hash=sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679 \ + --hash=sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8 \ + --hash=sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83 \ + --hash=sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6 \ + --hash=sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f \ + --hash=sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94 \ + --hash=sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51 \ + --hash=sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120 \ + --hash=sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039 \ + --hash=sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1 \ + --hash=sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05 \ + --hash=sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb \ + --hash=sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144 \ + --hash=sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa \ + --hash=sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a \ + --hash=sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99 \ + --hash=sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928 \ + --hash=sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d \ + --hash=sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3 \ + --hash=sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434 \ + --hash=sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86 \ + --hash=sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46 \ + --hash=sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319 \ + --hash=sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67 \ + --hash=sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c \ + --hash=sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169 \ + --hash=sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c \ + --hash=sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59 \ + --hash=sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107 \ + --hash=sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4 \ + --hash=sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a \ + --hash=sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb \ + --hash=sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f \ + --hash=sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769 \ + --hash=sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432 \ + --hash=sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090 \ + --hash=sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764 \ + --hash=sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d \ + --hash=sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4 \ + --hash=sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b \ + --hash=sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d \ + --hash=sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543 \ + --hash=sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24 \ + --hash=sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5 \ + --hash=sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b \ + --hash=sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d \ + --hash=sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b \ + --hash=sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6 \ + --hash=sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735 \ + --hash=sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e \ + --hash=sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28 \ + --hash=sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3 \ + --hash=sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401 \ + --hash=sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6 \ + --hash=sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d # via aiohttp -zipp==3.23.0 +zipp==3.23.0 \ + --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ + --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 # via importlib-metadata diff --git a/setup_gemini.py b/setup_gemini.py index a7c95fbe..655fc1ca 100644 --- a/setup_gemini.py +++ b/setup_gemini.py @@ -60,7 +60,7 @@ # Use --prefix ~/.local so npm installs directly into ~/.local/bin (avoids EACCES on /usr/local) npm_prefix = str(home / ".local") gemini_version = get_npm_version("@google/gemini-cli") - gemini_pkg = f"@google/gemini-cli@{gemini_version}" if gemini_version else "@google/gemini-cli@nightly" + gemini_pkg = f"@google/gemini-cli@{gemini_version}" if gemini_version else "@google/gemini-cli@latest" print(f"Installing {gemini_pkg}...") result = subprocess.run( ["npm", "install", "-g", f"--prefix={npm_prefix}", gemini_pkg], From be3b9edfdb7d1b081b8656465995825d58b69543 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Thu, 12 Mar 2026 13:52:27 -0400 Subject: [PATCH 125/382] Revert "chore: move landing page to docs/site/index.html" This reverts commit ea25749e62be037b7cca1db1fd38c7765f5e54fb. --- docs/{site => }/index.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{site => }/index.html (100%) diff --git a/docs/site/index.html b/docs/index.html similarity index 100% rename from docs/site/index.html rename to docs/index.html From cf2e3396a3a4d0155e5e268a44857f98fc48ee8c Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Thu, 12 Mar 2026 13:52:27 -0400 Subject: [PATCH 126/382] Revert "feat: add CoDA landing page for GitHub Pages" This reverts commit 55f63224cebb5d3cc7ed8ed9161ed9f09ccbc274. --- docs/index.html | 804 ------------------------------------------------ 1 file changed, 804 deletions(-) delete mode 100644 docs/index.html diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 4463accf..00000000 --- a/docs/index.html +++ /dev/null @@ -1,804 +0,0 @@ - - - - - - CoDA — Co-Working Developer Agents on Databricks Apps - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- -
-

CoDA

-

Co-Working Developer Agents

- -

- Four AI coding agents. - One Databricks App. Three steps to running. -

-

- Claude Code, Codex, Gemini CLI, and OpenCode — configured for Unity Catalog, AI Gateway, and Workspace files out of the box. -

- -
- - Fork on GitHub - - - See how it works - -
- - -
-
-
-
-
-
- - - - -
-
-
-
~/workspace/project
-
- -
-
-
-
- -
- - - - -
-
-
-

Four Agents, One Terminal

-

Pick the right model for the job

-

Different models see different things. Switch agents with a click — they share the same workspace, the same data, the same Databricks context.

-
- -
- -
-
-
- -
-
-

Claude Code

-

Anthropic

-
-
-

databricks-claude-opus-4-6

-

Deep Databricks skills + useful MCP servers. The most deeply integrated agent.

-
- - -
-
-
- -
-
-

Codex

-

OpenAI

-
-
-

databricks-codex

-

OpenAI's reasoning engine. Excels at multi-step code generation and refactoring.

-
- - -
-
-
- -
-
-

Gemini CLI

-

Google

-
-
-

databricks-gemini-2.5-pro

-

Google's multimodal agent. Vision, long context, and deep reasoning.

-
- - -
-
-
- -
-
-

OpenCode

-

Open Source

-
-
-

multi-provider

-

Open-source, multi-provider. Use any model, any backend, full transparency.

-
-
-
-
- -
- - - - -
-
-
-

What Ships in the Box

-

Everything is wired together

-

Skills, servers, and integrations keep growing. Here's what's configured today.

-
- -
-
-
- -
-

Databricks Skills

-

Pipelines, dashboards, Unity Catalog, Lakebase — a growing library.

-
- -
-
- -
-

MCP Servers

-

DeepWiki, Exa, and more — wired into every agent and growing.

-
- -
-
- -
-

MLflow Tracing

-

Every agent session auto-traced, queryable via Genie.

-
- -
-
- -
-

Workspace Sync

-

git commit auto-pushes to your Workspace path.

-
- -
-
- -
-

AI Gateway Routing

-

One config, any model, full cost tracking.

-
- -
-
- -
-

Terminal Themes

-

Dracula, Nord, Monokai, and more. Pick your vibe.

-
- -
-
- -
-

Voice & Image Input

-

Dictate or drag-drop images into the terminal.

-
- -
-
- -
-

Supply Chain Security

-

All deps SHA-pinned. Weekly CVE audits via GitHub Actions.

-
-
-
-
- -
- - - - -
-
-
-

Why Databricks Apps?

-

You bring the code.
Databricks brings the infra.

-

Running coding agents locally means juggling API keys, model access, and governance. Databricks Apps handles all of that.

-
- -
-
-
- -
-

Identity & Auth

-

- Your workspace token flows through. No API key juggling. Single-user isolation by default. -

-
- -
-
- -
-

AI Gateway

-

- Route agents to any foundation model — Claude, GPT, Gemini — through one gateway. Usage tracked, costs governed. -

-
- -
-
- -
-

Data & Governance

-

- Unity Catalog, MLflow, Workspace files — agents have native access to your entire lakehouse. -

-
-
- -
-

- Databricks Apps gives coding agents what they actually need: identity, models, data, and governance. CoDA just wires it all together. -

-
-
-
- -
- - - - -
-
-
-
-
- -
-
-

Need something more specialized?

-

CoDA agents are general-purpose. Genie Code is bespoke.

-

- The agents in CoDA — Claude Code, Codex, Gemini CLI, OpenCode — are general-purpose coding agents that work across any codebase. They're great for broad software engineering tasks. -

-

- But if you need an agent that deeply understands your lakehouse — your table schemas, column lineage, governance policies, pipeline failures — Genie Code is purpose-built for that. It's Databricks' autonomous AI agent for data engineering, data science, and ML work, with native Unity Catalog context that general-purpose agents can't match. -

- - Read the Genie Code announcement - -
-
-
-
-
- -
- - - - -
-
-
-

Get Started

-

Three steps. No Terraform.

-
- -
-
-
1
-
-

Fork the template

-

One click on GitHub. You get the full CoDA setup — agents, skills, MCP servers, themes, and CI.

-
-
- -
-
2
-
-

Create a Databricks App

-

Connect your repo, pick a name. Databricks handles compute, networking, and identity.

-
-
- -
-
3
-
-

Set your token, deploy

-

Add your Databricks token as a secret, hit deploy. Agents start with the app.

-
-
-
- -
-
- Dockerfile - Terraform - Kubernetes - app.yaml -
- - Fork on GitHub - -
-
-
- -
- - - - - - - - - - - - From 906f637adfe9681dfc0a34be2d3ef0790f459c97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 14:10:46 -0400 Subject: [PATCH 127/382] chore(deps): bump actions/setup-python from 5.6.0 to 6.2.0 (#65) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.6.0 to 6.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/a26af69be951a213d495a4c3e4e4022e16d87065...a309ff8b426b58ec0e2a45f0f869d46889d02405) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dependency-audit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index f4746bee..ce3668c8 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" From 327e0f456d61325693c5fe8c380874edb8f1288c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 14:17:53 -0400 Subject: [PATCH 128/382] chore(deps): bump actions/checkout from 4.3.1 to 6.0.2 (#64) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dependabot: bump actions/checkout v4.3.1 → v6.0.2 (SHA-pinned) --- .github/workflows/dependency-audit.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index ce3668c8..b152db0b 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4bc53104..cc876d59 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 From 0db41a6670bfbb6945572bf0401ad0a7f9748b3d Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Wed, 11 Mar 2026 14:59:44 +1100 Subject: [PATCH 129/382] feat: bundle TDD subagents for Claude Code in app Copy prd-writer, test-generator, implementer, and build-feature agent definitions to ~/.claude/agents/ during setup. Stripped model overrides so agents inherit the Databricks model serving endpoint. Co-Authored-By: Claude Opus 4.6 --- agents/build-feature.md | 66 ++++++++++++++++++++++++++++++++ agents/implementer.md | 59 +++++++++++++++++++++++++++++ agents/prd-writer.md | 81 ++++++++++++++++++++++++++++++++++++++++ agents/test-generator.md | 56 +++++++++++++++++++++++++++ setup_claude.py | 19 +++++++++- 5 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 agents/build-feature.md create mode 100644 agents/implementer.md create mode 100644 agents/prd-writer.md create mode 100644 agents/test-generator.md diff --git a/agents/build-feature.md b/agents/build-feature.md new file mode 100644 index 00000000..9a357776 --- /dev/null +++ b/agents/build-feature.md @@ -0,0 +1,66 @@ +--- +name: build-feature +description: End-to-end feature builder. Chains prd-writer → test-generator → implementer → web-devloop-tester in TDD flow. Use when asked to "build", "create", or "implement" a feature from scratch. Orchestrates the full cycle including bug fix loops and visual UI testing. +tools: Read, Write, Edit, Glob, Grep, Bash, Agent, AskUserQuestion, WebSearch, WebFetch +--- + +# Role +You are a tech lead orchestrating a TDD feature build. You coordinate four phases and handle failures. + +# Phase 1: PRD +1. Invoke yourself as a prd-writer: interview the user, write `docs/prd/.md` +2. Do NOT proceed until the user approves the PRD +3. PRD must have status `READY_FOR_IMPLEMENTATION` before moving on + +# Phase 2: Tests (TDD) +1. Read the approved PRD +2. Extract all Acceptance Criteria (AC-*) +3. Scan the codebase for test framework and conventions +4. Write failing tests that define the contract — one or more tests per AC +5. Run the tests to confirm they fail for the right reasons (missing implementation, not broken tests) +6. Update PRD status to `TESTS_WRITTEN` + +# Phase 3: Implementation +1. Read the PRD and all test files +2. Run the test suite to see current failures +3. Create an implementation plan, present it to the user for approval +4. Implement code to make tests pass, working through one group at a time +5. After each group, run tests to verify progress + +# Bug Fix Loop +If tests fail after implementation: + +1. Read the failure output carefully +2. Identify whether the bug is in the **test** or the **implementation** +3. If test is wrong (doesn't match PRD): fix the test +4. If implementation is wrong: fix the code +5. Re-run tests +6. **Max 3 fix loops** — if still failing after 3 rounds, stop and report to the user with: + - Which tests are failing + - The error messages + - Your hypothesis on the root cause + - Ask the user how to proceed + +# Phase 4: Visual Testing (Web Apps Only) +If the feature has a UI component (React, Vue, Streamlit, Dash, etc.): + +1. Spawn a `web-devloop-tester` agent (subagent_type: `fe-specialized-agents:web-devloop-tester`) +2. Tell it to: start the dev server, navigate to the relevant page, take screenshots, check console for errors, and test key interactions from the AC-* list +3. Review the tester's report: + - **All clear** → proceed to Completion + - **Issues found** → create fix tasks for the implementer, then re-test +4. **Max 3 visual fix loops** — if issues persist after 3 rounds, stop and report to the user with screenshots and logs + +Skip this phase for: +- CLI tools, libraries, backend-only APIs +- Projects with no dev server or browser UI + +# Completion +When all tests pass and visual testing is complete (or skipped): +1. Run the full test suite one final time +2. Update PRD status to `COMPLETE` +3. Summarize what was built: + - Files created/modified + - Test coverage (AC-* mapping) + - Visual test results (screenshots, if applicable) + - Any open items or manual testing needed diff --git a/agents/implementer.md b/agents/implementer.md new file mode 100644 index 00000000..2f6d0881 --- /dev/null +++ b/agents/implementer.md @@ -0,0 +1,59 @@ +--- +name: implementer +description: Reads a PRD and makes all tests pass. Implements code to satisfy the test suite written by test-generator. Use after test-generator has written failing tests. Runs tests iteratively until green. +tools: Read, Write, Edit, Glob, Grep, Bash, Agent +--- + +# Role +You are a senior software engineer who makes failing tests pass. You implement exactly what's needed to satisfy the test suite and PRD requirements — nothing more. + +# Startup +1. Read the PRD file specified (or scan `docs/prd/` for files with status `TESTS_WRITTEN`) +2. Read ALL test files listed in the PRD status section +3. Run the test suite to see the current failures +4. Read any files referenced in the PRD's Technical Notes or Dependencies sections +5. Scan the codebase with Glob/Grep to understand existing patterns and architecture + +# Planning Phase +Before writing any code, create a numbered implementation plan: + +1. List every failing test and what it expects +2. Group tests by module/component +3. Identify files to create or modify +4. Note the order of operations (what depends on what) +5. Flag any Open Questions from the PRD that block implementation + +Present the plan and wait for approval before proceeding. + +# Implementation Phase — Red-Green Loop +For each group of related tests: + +1. **Read the tests** — understand exactly what they expect +2. **Write minimal code** to make those tests pass +3. **Run tests** — check if they pass +4. **If tests fail** — read the error, fix the code, run again +5. **Repeat** until that group is green +6. **Commit** — use `git commit -m "message"` directly +7. Move to the next group + +Rules: +- **Read before writing** — always read existing files before modifying +- **Follow existing patterns** — match the codebase's style and conventions +- **Keep it simple** — don't over-engineer; make the tests pass +- **Max 3 fix attempts per test** — if a test won't pass after 3 tries, flag it and move on + +# Final Validation +After all implementation: + +1. Run the FULL test suite +2. If any tests still fail, attempt fixes (max 2 more rounds) +3. If tests still fail after retries, document the failures + +# Handoff +When complete, update the PRD status: + +> **Status: IMPLEMENTED** +> Commits: +> Test results: +> If all green: **Status: COMPLETE** +> If failures remain: **Status: NEEDS_REVIEW** with failure details diff --git a/agents/prd-writer.md b/agents/prd-writer.md new file mode 100644 index 00000000..baf4aa01 --- /dev/null +++ b/agents/prd-writer.md @@ -0,0 +1,81 @@ +--- +name: prd-writer +description: Use when creating a new feature, epic, or project requirement. Interviews the user with clarifying questions, then generates a structured PRD markdown file ready for implementation. Use proactively when asked about new features or "what should we build". +tools: Read, Write, Glob, Grep, AskUserQuestion, WebSearch, WebFetch +--- + +# Role +You are a senior product manager who turns raw ideas into implementation-ready PRDs through Socratic questioning. + +# Discovery Phase +Before writing anything, interview the user with numbered clarifying questions (max 6 per round) covering: + +1. **Problem** — What problem are we solving and who does it affect? +2. **Success metrics** — How will we know this worked? What are the acceptance criteria? +3. **Scope boundaries** — What is explicitly OUT of scope? +4. **Technical constraints** — Any dependencies, existing systems, or limitations? +5. **Priority & timeline** — How urgent is this? What's the desired delivery window? +6. **Edge cases** — What happens when things go wrong? Error states? + +Use AskUserQuestion to present these as structured questions. WAIT for answers before proceeding. Ask follow-up rounds if answers are vague or incomplete. + +# Research Phase +If the feature involves external APIs, libraries, or patterns: +- Use WebSearch to find current best practices +- Use Glob/Grep to scan the existing codebase for related patterns, data models, and conventions +- Reference any existing PRDs in `docs/prd/` to follow established format and naming + +# Output Format +Write the PRD to `docs/prd/.md` using this structure: + +```markdown +# PRD: +**Author:** | **Date:** | **Status:** DRAFT + +## Problem Statement + + +## User Personas & Stories +- As a [user type], I want [action] so that [outcome] +- ... + +## Functional Requirements +1. FR-1: +2. FR-2: ... + +## Non-Functional Requirements +1. NFR-1: +2. NFR-2: ... + +## Acceptance Criteria +1. AC-1: Given [context], when [action], then [result] +2. AC-2: ... + +## Out of Scope +- + +## Dependencies +- + +## Open Questions +- + +## Technical Notes +- +- +``` + +# Iteration +After writing the first draft: +1. Present a summary to the user +2. Ask if any sections need refinement +3. Update the PRD based on feedback +4. Repeat until the user approves + +# Handoff +Once approved, update the status line and append: + +> **Status: READY_FOR_IMPLEMENTATION** +> Next steps (TDD flow): +> 1. test-generator writes failing tests from the Acceptance Criteria +> 2. implementer makes all tests pass diff --git a/agents/test-generator.md b/agents/test-generator.md new file mode 100644 index 00000000..f2f2d21b --- /dev/null +++ b/agents/test-generator.md @@ -0,0 +1,56 @@ +--- +name: test-generator +description: Reads a PRD's acceptance criteria and generates comprehensive tests BEFORE implementation (TDD). Maps each AC-* criterion to one or more test cases. Tests should initially fail — that's expected. Use after prd-writer and BEFORE the implementer. +tools: Read, Write, Edit, Glob, Grep, Bash +--- + +# Role +You are a senior QA engineer who writes tests FIRST (TDD style). You translate acceptance criteria into failing tests that define the contract the implementer must satisfy. + +# Startup +1. Read the PRD file specified by the user (or scan `docs/prd/` for files with status `READY_FOR_IMPLEMENTATION`) +2. Extract all Acceptance Criteria (AC-*) +3. Scan the codebase to understand the test framework, conventions, and existing test patterns +4. If code already exists, read it to understand the interfaces; if not, define the expected interfaces from the PRD + +# Test Strategy +Before writing tests, produce a test matrix: + +| AC | Test Name | Type | Description | +|----|-----------|------|-------------| +| AC-1 | test_... | unit | ... | +| AC-1 | test_... | integration | ... | +| AC-2 | test_... | unit | ... | + +Every AC must have at least one test. Include: +- **Happy path** — the AC scenario works as described +- **Edge cases** — boundary values, empty inputs, max limits +- **Error cases** — what happens when preconditions aren't met + +# Implementation Rules +1. **Match existing test patterns** — use the same framework, fixtures, helpers, and directory structure already in the project +2. **Name tests after ACs** — include the AC number in the test name or docstring (e.g., `test_ac1_user_can_login`) +3. **Keep tests independent** — no test should depend on another test's state +4. **Test behavior, not implementation** — tests should survive refactoring +5. **Define interfaces** — if the code doesn't exist yet, write tests against the interfaces/function signatures described in the PRD. Import from expected module paths. + +# Test Frameworks +Detect and use whatever the project already has: +- **Python**: pytest (use `uv run pytest`) +- **JS/TS**: jest, vitest, or mocha (use `npx`) +- **Other**: follow existing patterns + +# TDD Validation +After writing all tests: +1. Run the test suite — **tests SHOULD fail** (no implementation yet) +2. Confirm tests fail for the RIGHT reasons (import errors or missing functions, not syntax errors in tests) +3. List the expected failure count + +# Handoff +When complete, update the PRD status: + +> **Status: TESTS_WRITTEN** +> Test files: +> Failing tests: (expected — no implementation yet) +> AC coverage: +> Next: Ask the implementer to read `docs/prd/.md` and make all tests pass diff --git a/setup_claude.py b/setup_claude.py index 128ef378..49e7e990 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -1,5 +1,6 @@ import os import json +import shutil import subprocess from pathlib import Path @@ -86,7 +87,23 @@ else: print(f"Claude Code CLI already installed at {claude_bin}") -# 4. Create projects directory +# 4. Copy subagent definitions to ~/.claude/agents/ +# These enable TDD workflow: prd-writer → test-generator → implementer → build-feature +agents_src = Path(__file__).parent / "agents" +agents_dst = claude_dir / "agents" +agents_dst.mkdir(exist_ok=True) + +if agents_src.exists(): + copied = [] + for agent_file in agents_src.glob("*.md"): + shutil.copy2(str(agent_file), str(agents_dst / agent_file.name)) + copied.append(agent_file.name) + if copied: + print(f"Subagents installed: {', '.join(copied)}") +else: + print("No agents directory found, skipping subagent setup") + +# 5. Create projects directory projects_dir = home / "projects" projects_dir.mkdir(exist_ok=True) print(f"Projects directory: {projects_dir}") From eb9de0a27c8c2b8e4ffdcc63c97690b213c63296 Mon Sep 17 00:00:00 2001 From: Marshall Krassenstein Date: Wed, 18 Mar 2026 20:03:41 -0400 Subject: [PATCH 130/382] Contributing guide --- CONTRIBUTING.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..0fce511d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,36 @@ +# Contributing to Claude Code on Databricks + +Thank you for your interest in contributing! We welcome and appreciate contributions from the community. + +## A Note on AI-Assisted Development + +In the age of agentic coding, **code review has become the bottleneck**. It's easier than ever to generate large volumes of code, but every PR still needs a human to carefully review, understand, and approve it. Please be thoughtful about what you submit — don't let an AI agent fire off a sprawling PR without you personally reviewing and understanding every change. + +**Before submitting, ask yourself:** + +- Do I understand every line of this diff? +- Is each change necessary and intentional? +- Have I kept the scope focused rather than letting an agent "improve" unrelated code? + +The easier your PR is to review, the faster it gets merged. Help us help you. 🤝 + +## How to Contribute + +1. **Fork & branch** — Fork the repository and create a descriptive branch for your work (e.g., `fix/websocket-reconnect` or `feat/session-timeout`). + +2. **Keep changes focused** — One logical change per PR. Avoid mixing refactors, formatting changes, or unrelated fixes into the same PR. + +3. **Write a test plan** — Ensure your changes pass existing tests before submitting. If your contribution can't be validated with unit tests, include screenshots or a video demonstrating the functionality in your PR description. + +4. **Deploy & verify on Dogfood** — Deploy your app on Dogfood and confirm it works as expected before opening a PR. + +5. **Open a pull request** — Write a clear description covering: + - **What** changed and **why** + - How to test or verify the changes + - Any known limitations or follow-up work + +## Code Review + +We review every PR and may request changes. Please don't take feedback personally — we're all working toward the same goal. The smaller and more focused your PR, the faster the turnaround. + +Thank you for contributing! From b7e496f1916b7785ddb4e059e8fcf06a8d82444f Mon Sep 17 00:00:00 2001 From: Marshall Date: Wed, 18 Mar 2026 20:09:11 -0400 Subject: [PATCH 131/382] Add a contributing guide --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0fce511d..98c95696 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,9 +20,9 @@ The easier your PR is to review, the faster it gets merged. Help us help you. 2. **Keep changes focused** — One logical change per PR. Avoid mixing refactors, formatting changes, or unrelated fixes into the same PR. -3. **Write a test plan** — Ensure your changes pass existing tests before submitting. If your contribution can't be validated with unit tests, include screenshots or a video demonstrating the functionality in your PR description. +3. **Write a test plan** — If your contribution can't be validated with unit tests, include screenshots or a video demonstrating the functionality in your PR description. -4. **Deploy & verify on Dogfood** — Deploy your app on Dogfood and confirm it works as expected before opening a PR. +4. **Deploy & verify on your workspace** — Deploy your app on your workspace (use Dogfood if you're a Brickster) and confirm it works as expected before opening a PR. 5. **Open a pull request** — Write a clear description covering: - **What** changed and **why** From b590830f0d4f2dc4feba54f07965b4c04d6604d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 05:03:55 -0400 Subject: [PATCH 132/382] chore(deps): bump softprops/action-gh-release from 2.5.0 to 2.6.1 (#75) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cc876d59..f7a0b5d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,7 +112,7 @@ jobs: git push origin "$TAG" - name: Create GitHub Release - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 + uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2 with: tag_name: "${{ steps.version.outputs.TAG }}" name: "${{ steps.version.outputs.TAG }}" From f819d5dbd7efc459f5bafc87aa5c6562e6c515e4 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Tue, 24 Mar 2026 12:42:28 -0400 Subject: [PATCH 133/382] fix: extend session linger to 24 hours (#77) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: extend session linger to 24 hours (closes #76) Sessions now survive for up to 24 hours of inactivity before cleanup reaps them. Active sessions with heartbeats live indefinitely — the timeout only applies to abandoned sessions. Cleanup interval bumped from 60s to 15min since frequent sweeps are unnecessary with a 24h window. * fix: close slave FD after Popen and set 32 MB upload limit - Close slave_fd in parent after Popen to prevent FD leak (fixes #78) - Set MAX_CONTENT_LENGTH to 32 MB aligned with Claude Code's 30 MB file limit (fixes #79) --- app.py | 6 +- static/index.html | 2 +- tests/test_session_linger.py | 189 +++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 tests/test_session_linger.py diff --git a/app.py b/app.py index 5526a6b6..1531aa30 100644 --- a/app.py +++ b/app.py @@ -30,8 +30,8 @@ APP_VERSION = '0.0.0' # Session timeout configuration -SESSION_TIMEOUT_SECONDS = 300 # No poll for 5 min = dead session -CLEANUP_INTERVAL_SECONDS = 60 # How often to check for stale sessions +SESSION_TIMEOUT_SECONDS = 86400 # No poll for 24 hours = dead session +CLEANUP_INTERVAL_SECONDS = 900 # Check for stale sessions every 15 min GRACEFUL_SHUTDOWN_WAIT = 3 # Seconds to wait after SIGHUP before SIGKILL # Logging setup @@ -40,6 +40,7 @@ app = Flask(__name__, static_folder='static', static_url_path='/static') app.secret_key = os.urandom(24) +app.config['MAX_CONTENT_LENGTH'] = 32 * 1024 * 1024 # 32 MB — aligned with Claude Code's 30 MB file limit # WebSocket support via Flask-SocketIO (simple-websocket transport, threading mode) socketio = SocketIO(app, async_mode='threading', cors_allowed_origins=[], logger=False, engineio_logger=False) @@ -690,6 +691,7 @@ def create_session(): env=shell_env, cwd=projects_dir ).pid + os.close(slave_fd) # Parent doesn't need the slave side; child inherited it session_id = str(uuid.uuid4()) diff --git a/static/index.html b/static/index.html index 0df0b68a..a74938e2 100644 --- a/static/index.html +++ b/static/index.html @@ -881,7 +881,7 @@

General

let socket = null; let wsConnected = false; let wsHeartbeatTimer = null; - const WS_HEARTBEAT_INTERVAL = 30000; // 30s — well within 5-min session timeout + const WS_HEARTBEAT_INTERVAL = 30000; // 30s — well within 24-hour session timeout function initWebSocket() { // Only init once; skip if Socket.IO client not loaded diff --git a/tests/test_session_linger.py b/tests/test_session_linger.py new file mode 100644 index 00000000..0b3f2aa3 --- /dev/null +++ b/tests/test_session_linger.py @@ -0,0 +1,189 @@ +"""Tests for 24-hour session linger (issue #76). + +Verifies that: +- SESSION_TIMEOUT_SECONDS is 86400 (24 hours) +- CLEANUP_INTERVAL_SECONDS is 900 (15 minutes) +- Sessions idle < 24h survive cleanup +- Sessions idle > 24h are reaped +- Warning fires at 80% of 24h (~19.2h) +- /api/status reports the correct timeout to the frontend +""" + +import time +from collections import deque +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _get_app(): + """Import app with initialize_app mocked out.""" + with mock.patch("app.initialize_app"): + import app as app_module + app_module.app.config["TESTING"] = True + return app_module + + +def _add_session(app_module, session_id, idle_seconds): + """Insert a fake session that has been idle for `idle_seconds`.""" + session = { + "master_fd": 999, + "pid": 12345, + "output_buffer": deque(maxlen=1000), + "last_poll_time": time.time() - idle_seconds, + "created_at": time.time() - idle_seconds - 60, + } + with app_module.sessions_lock: + app_module.sessions[session_id] = session + return session + + +def _cleanup(app_module, *session_ids): + with app_module.sessions_lock: + for sid in session_ids: + app_module.sessions.pop(sid, None) + + +# --------------------------------------------------------------------------- +# 1. Constants are set to 24-hour values +# --------------------------------------------------------------------------- + +class TestTimeoutConstants: + + def test_session_timeout_is_24_hours(self): + app_module = _get_app() + assert app_module.SESSION_TIMEOUT_SECONDS == 86400 + + def test_cleanup_interval_is_15_minutes(self): + app_module = _get_app() + assert app_module.CLEANUP_INTERVAL_SECONDS == 900 + + +# --------------------------------------------------------------------------- +# 2. Sessions survive well within the 24h window +# --------------------------------------------------------------------------- + +class TestSessionSurvival: + + def test_session_idle_1_hour_survives(self): + app_module = _get_app() + _add_session(app_module, "alive-1h", idle_seconds=3600) + try: + now = time.time() + with app_module.sessions_lock: + idle = now - app_module.sessions["alive-1h"]["last_poll_time"] + assert idle <= app_module.SESSION_TIMEOUT_SECONDS + assert "alive-1h" in app_module.sessions + finally: + _cleanup(app_module, "alive-1h") + + def test_session_idle_12_hours_survives(self): + app_module = _get_app() + _add_session(app_module, "alive-12h", idle_seconds=43200) + try: + now = time.time() + with app_module.sessions_lock: + idle = now - app_module.sessions["alive-12h"]["last_poll_time"] + assert idle <= app_module.SESSION_TIMEOUT_SECONDS + assert "alive-12h" in app_module.sessions + finally: + _cleanup(app_module, "alive-12h") + + def test_session_idle_23_hours_survives(self): + app_module = _get_app() + _add_session(app_module, "alive-23h", idle_seconds=82800) + try: + now = time.time() + with app_module.sessions_lock: + idle = now - app_module.sessions["alive-23h"]["last_poll_time"] + assert idle <= app_module.SESSION_TIMEOUT_SECONDS + assert "alive-23h" in app_module.sessions + finally: + _cleanup(app_module, "alive-23h") + + +# --------------------------------------------------------------------------- +# 3. Sessions past 24h are reaped by cleanup +# --------------------------------------------------------------------------- + +class TestSessionReaping: + + def test_session_idle_25_hours_is_reaped(self): + app_module = _get_app() + _add_session(app_module, "stale-25h", idle_seconds=90000) + try: + stale = [] + now = time.time() + with app_module.sessions_lock: + for sid, s in app_module.sessions.items(): + if sid != "stale-25h": + continue + idle = now - s["last_poll_time"] + if idle > app_module.SESSION_TIMEOUT_SECONDS: + stale.append(sid) + assert "stale-25h" in stale + finally: + _cleanup(app_module, "stale-25h") + + def test_session_idle_exactly_24h_plus_1s_is_reaped(self): + app_module = _get_app() + _add_session(app_module, "stale-boundary", idle_seconds=86401) + try: + now = time.time() + with app_module.sessions_lock: + idle = now - app_module.sessions["stale-boundary"]["last_poll_time"] + assert idle > app_module.SESSION_TIMEOUT_SECONDS + finally: + _cleanup(app_module, "stale-boundary") + + +# --------------------------------------------------------------------------- +# 4. Warning fires at 80% (~19.2 hours) +# --------------------------------------------------------------------------- + +class TestTimeoutWarning: + + def test_no_warning_at_18_hours(self): + app_module = _get_app() + _add_session(app_module, "warn-18h", idle_seconds=64800) + try: + warning_threshold = app_module.SESSION_TIMEOUT_SECONDS * 0.8 + now = time.time() + with app_module.sessions_lock: + idle = now - app_module.sessions["warn-18h"]["last_poll_time"] + assert idle < warning_threshold + finally: + _cleanup(app_module, "warn-18h") + + def test_warning_at_20_hours(self): + app_module = _get_app() + _add_session(app_module, "warn-20h", idle_seconds=72000) + try: + warning_threshold = app_module.SESSION_TIMEOUT_SECONDS * 0.8 + now = time.time() + with app_module.sessions_lock: + s = app_module.sessions["warn-20h"] + idle = now - s["last_poll_time"] + assert idle > warning_threshold + assert idle <= app_module.SESSION_TIMEOUT_SECONDS + finally: + _cleanup(app_module, "warn-20h") + + +# --------------------------------------------------------------------------- +# 5. /api/status reports 86400 to the frontend +# --------------------------------------------------------------------------- + +class TestStatusEndpoint: + + def test_health_reports_24h_timeout(self): + app_module = _get_app() + client = app_module.app.test_client() + with mock.patch.object(app_module, "check_authorization", return_value=(True, "test-user")): + resp = client.get("/health") + body = resp.get_json() + assert body["session_timeout_seconds"] == 86400 From 055c2e77850d394c324f3e27208a769f8cc3a3c6 Mon Sep 17 00:00:00 2001 From: Marshall Date: Tue, 24 Mar 2026 12:56:20 -0400 Subject: [PATCH 134/382] Update contribution guidelines for clarity Added a suggestion to break larger PRs into smaller commits. --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98c95696..af41ba61 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to Claude Code on Databricks +# Contributing to Coding Agents on Databricks Thank you for your interest in contributing! We welcome and appreciate contributions from the community. @@ -18,7 +18,7 @@ The easier your PR is to review, the faster it gets merged. Help us help you. 1. **Fork & branch** — Fork the repository and create a descriptive branch for your work (e.g., `fix/websocket-reconnect` or `feat/session-timeout`). -2. **Keep changes focused** — One logical change per PR. Avoid mixing refactors, formatting changes, or unrelated fixes into the same PR. +2. **Keep changes focused** — One logical change per PR. Avoid mixing refactors, formatting changes, or unrelated fixes into the same PR. Try to break larger PRs into small commits. 3. **Write a test plan** — If your contribution can't be validated with unit tests, include screenshots or a video demonstrating the functionality in your PR description. From 0865265a27556304ea733f9f034f4e7bf6998a83 Mon Sep 17 00:00:00 2001 From: Marshall Date: Tue, 24 Mar 2026 12:56:52 -0400 Subject: [PATCH 135/382] Update title in CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af41ba61..302b215f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to Coding Agents on Databricks +# Contributing to Coding Agents on Databricks Apps Thank you for your interest in contributing! We welcome and appreciate contributions from the community. From e7e7e771c140fd3a329dca8033150cf70db9446e Mon Sep 17 00:00:00 2001 From: David O'Keeffe <17697537+dgokeeffe@users.noreply.github.com> Date: Wed, 25 Mar 2026 03:59:09 +1100 Subject: [PATCH 136/382] fix: Windows copy-paste, resize debounce, CLI upgrades (#72) * fix: debounce resize handler, add scrollback config Prevents fitAddon.fit() from thrashing scroll position on every resize pixel. Adds explicit scrollback and scrollOnUserInput. Co-authored-by: Isaac * fix: always upgrade Claude Code CLI on startup Previously skipped install if binary existed, leaving stale versions across redeployments. Co-authored-by: Isaac * fix: enable Ctrl+C/V copy-paste on Windows terminals xterm.js intercepts Ctrl+V/C as raw control characters on non-Mac platforms. Added attachCustomKeyEventHandler to let the browser handle Ctrl+V (paste), Ctrl+C (copy when text selected), and Ctrl+Shift+C/V. Also added clipboard section to shortcuts help with platform-aware labels (Cmd on Mac, Ctrl on Windows) and upload toast for image paste. Co-authored-by: Isaac * feat: upgrade Databricks CLI to latest at startup Runtime image ships an older CLI (v0.251.0). Added a setup step that fetches the latest release from GitHub API and installs it to ~/.local/bin, same pattern as the GitHub CLI install. Co-authored-by: Isaac * refactor: extract gh and databricks CLI installs to .sh scripts Move inline bash from app.py into install_gh.sh and install_databricks_cli.sh, matching the install_micro.sh pattern. The gh script now fetches the latest version dynamically instead of hardcoding v2.74.1, and fixes a bug in the auth wrapper where the `gh auth login` handler fell through to `exec gh.real "$@"` with shifted args (added `exit 0`). Co-authored-by: Isaac --- app.py | 83 +++++++++++++++++++++++++++------------ install_databricks_cli.sh | 26 ++++++++++++ install_gh.sh | 45 +++++++++++++++++++++ setup_claude.py | 54 ++++++++++++++----------- static/index.html | 40 ++++++++++++++++++- 5 files changed, 197 insertions(+), 51 deletions(-) create mode 100644 install_databricks_cli.sh create mode 100644 install_gh.sh diff --git a/app.py b/app.py index 1531aa30..89fdcdd6 100644 --- a/app.py +++ b/app.py @@ -21,6 +21,13 @@ from utils import ensure_https +# Sanitize DATABRICKS_TOKEN early — the platform sometimes injects trailing +# newlines / whitespace which causes auth failures. Cleaning it here prevents +# the agent from "fixing" it in the terminal and leaking the raw token. +_raw_token = os.environ.get("DATABRICKS_TOKEN", "") +if _raw_token != _raw_token.strip(): + os.environ["DATABRICKS_TOKEN"] = _raw_token.strip() + # App version (single source of truth: pyproject.toml) _pyproject_file = os.path.join(os.path.dirname(__file__), 'pyproject.toml') try: @@ -84,6 +91,8 @@ def handle_sigterm(signum, frame): "steps": [ {"id": "git", "label": "Configuring git identity", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "micro", "label": "Installing micro editor", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "gh", "label": "Installing GitHub CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "dbcli", "label": "Upgrading Databricks CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "proxy", "label": "Starting content-filter proxy", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "claude", "label": "Configuring Claude CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "codex", "label": "Configuring Codex CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, @@ -118,6 +127,11 @@ def _run_step(step_id, command): env = os.environ.copy() if not env.get("HOME") or env["HOME"] == "/": env["HOME"] = "/app/python/source_code" + home = env.get("HOME", "/app/python/source_code") + # Ensure uv and other tools in ~/.local/bin are on PATH + local_bin = os.path.join(home, ".local", "bin") + if local_bin not in env.get("PATH", ""): + env["PATH"] = f"{local_bin}:{env.get('PATH', '')}" env.pop("DATABRICKS_CLIENT_ID", None) env.pop("DATABRICKS_CLIENT_SECRET", None) @@ -199,14 +213,14 @@ def _setup_git_config(): f.write('\n') f.write('echo "[post-commit] $(date +%H:%M:%S) syncing $REPO_ROOT" >> "$SYNC_LOG"\n') f.write('\n') - f.write('# Use venv python directly (avoids fragile source activate)\n') - f.write('VENV_PYTHON="/app/python/source_code/.venv/bin/python"\n') - f.write('SYNC_SCRIPT="/app/python/source_code/sync_to_workspace.py"\n') + f.write('# Use uv run so sync script gets the correct Python + deps\n') + f.write('APP_DIR="/app/python/source_code"\n') + f.write('SYNC_SCRIPT="$APP_DIR/sync_to_workspace.py"\n') f.write('\n') - f.write('if [ -x "$VENV_PYTHON" ] && [ -f "$SYNC_SCRIPT" ]; then\n') - f.write(' nohup "$VENV_PYTHON" "$SYNC_SCRIPT" "$REPO_ROOT" >> "$SYNC_LOG" 2>&1 & disown\n') + f.write('if [ -f "$SYNC_SCRIPT" ]; then\n') + f.write(' nohup uv run --project "$APP_DIR" python "$SYNC_SCRIPT" "$REPO_ROOT" >> "$SYNC_LOG" 2>&1 & disown\n') f.write('else\n') - f.write(' echo "[post-commit] $(date +%H:%M:%S) SKIP: venv=$VENV_PYTHON script=$SYNC_SCRIPT" >> "$SYNC_LOG"\n') + f.write(' echo "[post-commit] $(date +%H:%M:%S) SKIP: sync script not found" >> "$SYNC_LOG"\n') f.write('fi\n') os.chmod(post_commit, 0o755) logger.info(f"Post-commit hook written to {post_commit}") @@ -253,19 +267,24 @@ def run_setup(): _run_step("micro", ["bash", "-c", "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) + _run_step("gh", ["bash", "install_gh.sh"]) + + # --- Upgrade Databricks CLI (runtime image ships an older version) --- + _run_step("dbcli", ["bash", "install_databricks_cli.sh"]) + # --- Content-filter proxy (must be running before OpenCode starts) --- # Sanitizes requests/responses between OpenCode and Databricks # (see OpenCode #5028, docs/plans/2026-03-11-litellm-empty-content-blocks-design.md) - _run_step("proxy", ["python", "setup_proxy.py"]) + _run_step("proxy", ["uv", "run", "python", "setup_proxy.py"]) # --- Parallel agent setup (all independent of each other) --- parallel_steps = [ - ("claude", ["python", "setup_claude.py"]), - ("codex", ["python", "setup_codex.py"]), - ("opencode", ["python", "setup_opencode.py"]), - ("gemini", ["python", "setup_gemini.py"]), - ("databricks", ["python", "setup_databricks.py"]), - ("mlflow", ["python", "setup_mlflow.py"]), + ("claude", ["uv", "run", "python", "setup_claude.py"]), + ("codex", ["uv", "run", "python", "setup_codex.py"]), + ("opencode", ["uv", "run", "python", "setup_opencode.py"]), + ("gemini", ["uv", "run", "python", "setup_gemini.py"]), + ("databricks", ["uv", "run", "python", "setup_databricks.py"]), + ("mlflow", ["uv", "run", "python", "setup_mlflow.py"]), ] with ThreadPoolExecutor(max_workers=len(parallel_steps)) as executor: @@ -488,7 +507,7 @@ def read_pty_output(session_id, fd): if session_id not in sessions: break try: - readable, _, errors = select.select([fd], [], [fd], 0.5) + readable, _, errors = select.select([fd], [], [fd], 0.05) if readable or errors: output = os.read(fd, 4096) if not output: @@ -570,7 +589,10 @@ def cleanup_stale_sessions(): warning_threshold = SESSION_TIMEOUT_SECONDS * 0.8 with sessions_lock: - for session_id, session in sessions.items(): + session_snapshot = list(sessions.items()) + + for session_id, session in session_snapshot: + with session["lock"]: idle = now - session["last_poll_time"] if idle > SESSION_TIMEOUT_SECONDS: stale_sessions.append((session_id, session["pid"], session["master_fd"])) @@ -803,22 +825,31 @@ def get_output_batch(): outputs = {} now = time.time() + # Step 1: Resolve session refs under global lock (fast dict lookups only) + resolved = {} with sessions_lock: for sid in session_ids: - if sid not in sessions: - continue - session = sessions[sid] + if sid in sessions: + resolved[sid] = sessions[sid] + + # Step 2: Swap buffers under per-session locks (same pattern as get_output) + swapped = {} + for sid, session in resolved.items(): + with session["lock"]: session["last_poll_time"] = now - buffer = session["output_buffer"] - output = "".join(buffer) - buffer.clear() + old_buffer = session["output_buffer"] + session["output_buffer"] = deque(maxlen=1000) exited = session.get("exited", False) timeout_warning = session.pop("timeout_warning", False) - outputs[sid] = { - "output": output, - "exited": exited, - "timeout_warning": timeout_warning - } + swapped[sid] = (old_buffer, exited, timeout_warning) + + # Step 3: Join strings outside all locks + for sid, (old_buffer, exited, timeout_warning) in swapped.items(): + outputs[sid] = { + "output": "".join(old_buffer), + "exited": exited, + "timeout_warning": timeout_warning, + } return jsonify({"outputs": outputs, "shutting_down": shutting_down}) diff --git a/install_databricks_cli.sh b/install_databricks_cli.sh new file mode 100644 index 00000000..5409533c --- /dev/null +++ b/install_databricks_cli.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Install the latest Databricks CLI to ~/.local/bin. +# +# - Fetches the latest release tag from the GitHub API +# - Downloads and unzips the Linux amd64 binary +# - Prints the installed version + +set -euo pipefail + +INSTALL_DIR="$HOME/.local/bin" +mkdir -p "$INSTALL_DIR" + +# Fetch latest release tag +DB_CLI_VERSION=$(curl -fsSL "https://api.github.com/repos/databricks/cli/releases/latest" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))") + +echo "Installing Databricks CLI v${DB_CLI_VERSION}" + +curl -fsSL "https://github.com/databricks/cli/releases/download/v${DB_CLI_VERSION}/databricks_cli_${DB_CLI_VERSION}_linux_amd64.zip" \ + -o /tmp/dbcli.zip +unzip -o /tmp/dbcli.zip -d /tmp/dbcli +mv /tmp/dbcli/databricks "$INSTALL_DIR/databricks" +rm -rf /tmp/dbcli.zip /tmp/dbcli +chmod +x "$INSTALL_DIR/databricks" + +"$INSTALL_DIR/databricks" --version diff --git a/install_gh.sh b/install_gh.sh new file mode 100644 index 00000000..3cbe1b4e --- /dev/null +++ b/install_gh.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Install GitHub CLI (gh) to ~/.local/bin with an auth-login wrapper. +# +# - Fetches the latest 2.x release from the GitHub API +# - Installs to ~/.local/bin/gh.real +# - Creates a wrapper at ~/.local/bin/gh that intercepts `gh auth login` +# to skip interactive prompts (arrow-key menus break in xterm.js PTY) + +set -euo pipefail + +INSTALL_DIR="$HOME/.local/bin" +mkdir -p "$INSTALL_DIR" + +# Fetch latest release tag +GH_VERSION=$(curl -fsSL "https://api.github.com/repos/cli/cli/releases/latest" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))") + +echo "Installing GitHub CLI v${GH_VERSION}" + +curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \ + -o /tmp/gh.tar.gz +tar -xzf /tmp/gh.tar.gz -C /tmp +mv "/tmp/gh_${GH_VERSION}_linux_amd64/bin/gh" "$INSTALL_DIR/gh" +rm -rf /tmp/gh.tar.gz "/tmp/gh_${GH_VERSION}_linux_amd64" +chmod +x "$INSTALL_DIR/gh" + +# Set git protocol to HTTPS +"$INSTALL_DIR/gh" config set git_protocol https 2>/dev/null || true + +# Create wrapper that intercepts `gh auth login` to avoid interactive prompts +cat > "$INSTALL_DIR/gh.wrapper" << 'WRAPPER' +#!/bin/bash +if [ "$1" = "auth" ] && [ "$2" = "login" ]; then + shift 2 + printf "Y\\n" | ~/.local/bin/gh.real auth login -h github.com -p https -w --skip-ssh-key "$@" + exit 0 +fi +exec ~/.local/bin/gh.real "$@" +WRAPPER + +mv "$INSTALL_DIR/gh" "$INSTALL_DIR/gh.real" +mv "$INSTALL_DIR/gh.wrapper" "$INSTALL_DIR/gh" +chmod +x "$INSTALL_DIR/gh" + +echo "GitHub CLI v${GH_VERSION} installed to $INSTALL_DIR" diff --git a/setup_claude.py b/setup_claude.py index 49e7e990..b138f1c1 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -48,18 +48,29 @@ settings_path.write_text(json.dumps(settings, indent=2)) # 2. Write ~/.claude.json with onboarding skip AND MCP servers +mcp_servers = { + "deepwiki": { + "type": "http", + "url": "https://mcp.deepwiki.com/mcp" + }, + "exa": { + "type": "http", + "url": "https://mcp.exa.ai/mcp" + } +} + +# Auto-configure team-memory MCP if URL is provided +team_memory_url = os.environ.get("TEAM_MEMORY_MCP_URL", "").strip().rstrip("/") +if team_memory_url: + mcp_servers["team-memory"] = { + "type": "http", + "url": f"{team_memory_url}/mcp" + } + print(f"Team memory MCP configured: {team_memory_url}/mcp") + claude_json = { "hasCompletedOnboarding": True, - "mcpServers": { - "deepwiki": { - "type": "http", - "url": "https://mcp.deepwiki.com/mcp" - }, - "exa": { - "type": "http", - "url": "https://mcp.exa.ai/mcp" - } - } + "mcpServers": mcp_servers } claude_json_path = home / ".claude.json" @@ -72,20 +83,17 @@ local_bin = home / ".local" / "bin" claude_bin = local_bin / "claude" -if not claude_bin.exists(): - print("Installing Claude Code CLI...") - result = subprocess.run( - ["bash", "-c", "curl -fsSL https://claude.ai/install.sh | bash"], - env={**os.environ, "HOME": str(home)}, - capture_output=True, - text=True - ) - if result.returncode == 0: - print("Claude Code CLI installed successfully") - else: - print(f"CLI install warning: {result.stderr}") +print("Installing/upgrading Claude Code CLI...") +result = subprocess.run( + ["bash", "-c", "curl -fsSL https://claude.ai/install.sh | bash"], + env={**os.environ, "HOME": str(home)}, + capture_output=True, + text=True +) +if result.returncode == 0: + print("Claude Code CLI installed successfully") else: - print(f"Claude Code CLI already installed at {claude_bin}") + print(f"CLI install warning: {result.stderr}") # 4. Copy subagent definitions to ~/.claude/agents/ # These enable TDD workflow: prd-writer → test-generator → implementer → build-feature diff --git a/static/index.html b/static/index.html index a74938e2..9c5617f0 100644 --- a/static/index.html +++ b/static/index.html @@ -357,6 +357,10 @@

Panes

Close paneAlt+Shift+W
Next paneAlt+Shift+]
Previous paneAlt+Shift+[
+

Clipboard

+
CopyCtrl+C
+
PasteCtrl+V
+
Paste imagePaste from clipboard

General

SearchCtrl+Shift+F
Voice dictationAlt+V
@@ -379,6 +383,14 @@

General

- - diff --git a/tests/test_app_state.py b/tests/test_app_state.py new file mode 100644 index 00000000..0af32301 --- /dev/null +++ b/tests/test_app_state.py @@ -0,0 +1,82 @@ +"""Tests for app_state — persistent JSON at ~/.coda/app_state.json.""" + +import json +import os +import time +from unittest import mock + +import pytest + + +@pytest.fixture(autouse=True) +def isolated_state(tmp_path): + """Point app_state at a temp dir so tests don't touch real state.""" + state_dir = str(tmp_path / ".coda") + state_file = os.path.join(state_dir, "app_state.json") + with mock.patch("app_state._STATE_DIR", state_dir), \ + mock.patch("app_state._STATE_FILE", state_file): + yield state_file + + +class TestAppOwner: + def test_set_and_read_owner(self, isolated_state): + import app_state + app_state.set_app_owner("alice@example.com") + state = app_state.get_state() + assert state["app_owner"] == "alice@example.com" + assert "owner_resolved_at" in state + + def test_owner_persisted_to_disk(self, isolated_state): + import app_state + app_state.set_app_owner("bob@example.com") + with open(isolated_state) as f: + on_disk = json.load(f) + assert on_disk["app_owner"] == "bob@example.com" + + +class TestLastRotation: + def test_set_and_read_rotation(self, isolated_state): + import app_state + ts = time.time() + app_state.set_last_rotation("tid-abc", ts) + state = app_state.get_state() + assert state["last_token_id"] == "tid-abc" + assert state["last_rotation_time"] == ts + assert "last_rotation_iso" in state + + def test_get_last_rotation_time(self, isolated_state): + import app_state + assert app_state.get_last_rotation_time() is None + ts = time.time() + app_state.set_last_rotation("tid-xyz", ts) + assert app_state.get_last_rotation_time() == ts + + +class TestMerge: + def test_owner_and_rotation_coexist(self, isolated_state): + import app_state + app_state.set_app_owner("carol@example.com") + app_state.set_last_rotation("tid-123", time.time()) + state = app_state.get_state() + assert state["app_owner"] == "carol@example.com" + assert state["last_token_id"] == "tid-123" + + def test_file_permissions(self, isolated_state): + import app_state + import stat + app_state.set_app_owner("dave@example.com") + mode = stat.S_IMODE(os.stat(isolated_state).st_mode) + assert mode == 0o600 + + +class TestCorruptFile: + def test_corrupt_json_returns_empty(self, isolated_state): + import app_state + os.makedirs(os.path.dirname(isolated_state), exist_ok=True) + with open(isolated_state, "w") as f: + f.write("{bad json") + assert app_state.get_state() == {} + + def test_missing_file_returns_empty(self, isolated_state): + import app_state + assert app_state.get_state() == {} diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 4dd1b06b..1b4133a5 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -27,6 +27,7 @@ def _create_fake_session(app_module, session_id="test-session-123", **overrides) "output_buffer": deque(maxlen=1000), "last_poll_time": time.time() - 60, # 60s ago "created_at": time.time(), + "lock": __import__("threading").Lock(), } session.update(overrides) with app_module.sessions_lock: diff --git a/tests/test_pat_rotation_integration.py b/tests/test_pat_rotation_integration.py new file mode 100644 index 00000000..35b2a6eb --- /dev/null +++ b/tests/test_pat_rotation_integration.py @@ -0,0 +1,71 @@ +"""Integration test: PATRotator wired into app.""" + +import os +from unittest import mock + + +class TestPATRotatorIntegration: + + def test_app_has_pat_rotator(self): + with mock.patch("app.initialize_app"): + import app as app_module + assert hasattr(app_module, "pat_rotator") + + def test_pat_rotator_is_correct_type(self): + with mock.patch("app.initialize_app"): + import app as app_module + from pat_rotator import PATRotator + assert isinstance(app_module.pat_rotator, PATRotator) + + +class TestPATStatusEndpoint: + def test_pat_status_no_token(self): + with mock.patch("app.initialize_app"): + import app as app_module + app_module.app.config["TESTING"] = True + client = app_module.app.test_client() + + original = os.environ.pop("DATABRICKS_TOKEN", None) + try: + resp = client.get("/api/pat-status") + assert resp.status_code == 200 + data = resp.get_json() + assert data["configured"] is False + assert data["valid"] is False + finally: + if original: + os.environ["DATABRICKS_TOKEN"] = original + + def test_configure_pat_empty_token(self): + with mock.patch("app.initialize_app"): + import app as app_module + app_module.app.config["TESTING"] = True + client = app_module.app.test_client() + + resp = client.post("/api/configure-pat", json={"token": ""}) + assert resp.status_code == 400 + + +class TestPATStatusAccessible: + def test_pat_status_skips_auth(self): + """pat-status endpoint should be accessible without auth.""" + with mock.patch("app.initialize_app"): + import app as app_module + app_module.app.config["TESTING"] = True + app_module.app_owner = "owner@example.com" + client = app_module.app.test_client() + + resp = client.get("/api/pat-status") + assert resp.status_code == 200 # not 403 + + def test_configure_pat_skips_auth(self): + """configure-pat endpoint should be accessible without auth.""" + with mock.patch("app.initialize_app"): + import app as app_module + app_module.app.config["TESTING"] = True + app_module.app_owner = "owner@example.com" + client = app_module.app.test_client() + + # Should get 400 (bad request) not 403 (unauthorized) + resp = client.post("/api/configure-pat", json={"token": ""}) + assert resp.status_code == 400 diff --git a/tests/test_pat_rotator.py b/tests/test_pat_rotator.py new file mode 100644 index 00000000..40b412be --- /dev/null +++ b/tests/test_pat_rotator.py @@ -0,0 +1,400 @@ +"""Tests for PATRotator — short-lived PAT auto-rotation. + +Covers: rotation logic, token persistence, lifecycle management, +and logging output. +""" + +import logging +import os +import stat +import threading +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _mock_create_response(token_value="dapi-new-token-abc", token_id="tid-new-123", + status_code=200): + """Build a mock requests.Response for token/create.""" + resp = mock.MagicMock() + resp.status_code = status_code + if status_code == 200: + resp.json.return_value = { + "token_value": token_value, + "token_info": {"token_id": token_id}, + } + else: + resp.text = "error payload" + return resp + + +def _mock_delete_response(status_code=200): + """Build a mock requests.Response for token/delete.""" + resp = mock.MagicMock() + resp.status_code = status_code + resp.text = "delete payload" + return resp + + +def _make_rotator(**kwargs): + """Create a PATRotator with sane test defaults.""" + from pat_rotator import PATRotator + defaults = dict( + host="https://test.databricks.com", + rotation_interval=1, + token_lifetime=7200, + session_count_fn=lambda: 1, # default: pretend 1 active session + ) + defaults.update(kwargs) + return PATRotator(**defaults) + + +# --------------------------------------------------------------------------- +# 1. PAT Rotation — mint + revoke logic +# --------------------------------------------------------------------------- + +class TestPATRotation: + """Core rotation: mint new token, revoke old, handle failures.""" + + @mock.patch("pat_rotator.requests.post") + def test_mint_new_and_revoke_old(self, mock_post): + """Successful rotation: new token minted, old token revoked.""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-new", token_id="tid-new"), + _mock_delete_response(status_code=200), + ] + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-old" + + result = rotator._rotate_once() + + assert result is True + assert rotator.token == "dapi-new" + assert rotator._current_token_id == "tid-new" + # Two API calls: create + delete + assert mock_post.call_count == 2 + # Verify delete was called with old token id + delete_call = mock_post.call_args_list[1] + assert delete_call[1]["json"]["token_id"] == "tid-old" + + @mock.patch("pat_rotator.requests.post") + def test_create_failure_returns_false(self, mock_post): + """When token creation fails (non-200), rotation returns False.""" + mock_post.return_value = _mock_create_response(status_code=403) + rotator = _make_rotator() + rotator._current_token = "dapi-old" + + result = rotator._rotate_once() + + assert result is False + # Token should remain unchanged + assert rotator.token == "dapi-old" + + @mock.patch("pat_rotator.requests.post") + def test_create_request_exception_returns_false(self, mock_post): + """When create request raises an exception, rotation returns False.""" + import requests + mock_post.side_effect = requests.RequestException("network error") + rotator = _make_rotator() + rotator._current_token = "dapi-old" + + result = rotator._rotate_once() + + assert result is False + assert rotator.token == "dapi-old" + + @mock.patch("pat_rotator.requests.post") + def test_continues_if_revoke_fails(self, mock_post): + """New token is kept even when old token revocation fails.""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-new", token_id="tid-new"), + _mock_delete_response(status_code=500), + ] + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-old" + + result = rotator._rotate_once() + + assert result is True + assert rotator.token == "dapi-new" + + @mock.patch("pat_rotator.requests.post") + def test_first_rotation_no_old_token(self, mock_post): + """First rotation has no old token to revoke — should still succeed.""" + mock_post.return_value = _mock_create_response( + token_value="dapi-first", token_id="tid-first" + ) + rotator = _make_rotator() + rotator._current_token = "dapi-bootstrap" + rotator._current_token_id = None # no old token id + + result = rotator._rotate_once() + + assert result is True + assert rotator.token == "dapi-first" + # Only one API call (create), no delete + assert mock_post.call_count == 1 + + def test_no_token_returns_false(self): + """Rotation is a no-op when no current token exists.""" + rotator = _make_rotator() + rotator._current_token = None + + result = rotator._rotate_once() + + assert result is False + + +# --------------------------------------------------------------------------- +# 2. Token Persistence — env var + .databrickscfg +# --------------------------------------------------------------------------- + +class TestTokenPersistence: + """Token is persisted to env var and config file.""" + + @mock.patch("pat_rotator.requests.post") + def test_updates_env_var(self, mock_post): + """DATABRICKS_TOKEN env var is updated after rotation.""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-env-test", token_id="tid-env"), + _mock_delete_response(), + ] + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-old" + + with mock.patch.dict(os.environ, {"DATABRICKS_TOKEN": "dapi-old"}): + rotator._rotate_once() + assert os.environ["DATABRICKS_TOKEN"] == "dapi-env-test" + + @mock.patch("pat_rotator.requests.post") + def test_writes_databrickscfg(self, mock_post, tmp_path): + """Rotation writes a valid .databrickscfg file.""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-cfg-test", token_id="tid-cfg"), + _mock_delete_response(), + ] + cfg_path = str(tmp_path / ".databrickscfg") + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-old" + rotator._databrickscfg_path = cfg_path + + rotator._rotate_once() + + content = open(cfg_path).read() + assert "[DEFAULT]" in content + assert "token = dapi-cfg-test" in content + assert "host = https://test.databricks.com" in content + + @mock.patch("pat_rotator.requests.post") + def test_databrickscfg_permissions(self, mock_post, tmp_path): + """Config file should have 0o600 permissions (owner read/write only).""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-perm", token_id="tid-perm"), + _mock_delete_response(), + ] + cfg_path = str(tmp_path / ".databrickscfg") + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-old" + rotator._databrickscfg_path = cfg_path + + rotator._rotate_once() + + mode = stat.S_IMODE(os.stat(cfg_path).st_mode) + assert mode == 0o600 + + +# --------------------------------------------------------------------------- +# 3. Rotator Lifecycle — start / stop / daemon thread +# --------------------------------------------------------------------------- + +class TestRotatorLifecycle: + """Start/stop behavior and daemon thread management.""" + + def test_starts_daemon_thread(self): + """start() launches a daemon thread named 'pat-rotation'.""" + rotator = _make_rotator() + rotator._current_token = "dapi-lifecycle" + # Prevent actual rotation by making interval very long + rotator._rotation_interval = 9999 + + rotator.start() + try: + assert rotator._thread is not None + assert rotator._thread.is_alive() + assert rotator._thread.daemon is True + assert rotator._thread.name == "pat-rotation" + finally: + rotator.stop() + rotator._thread.join(timeout=2) + + def test_no_start_without_token(self): + """start() does nothing when no token is configured.""" + rotator = _make_rotator() + rotator._current_token = None + + rotator.start() + + assert rotator._thread is None + + def test_stop_signals_thread(self): + """stop() sets the stop event so the thread exits.""" + rotator = _make_rotator() + rotator._current_token = "dapi-stop-test" + rotator._rotation_interval = 9999 + + rotator.start() + rotator.stop() + rotator._thread.join(timeout=3) + + assert not rotator._thread.is_alive() + + def test_idempotent_start(self): + """Calling start() twice does not create a second thread.""" + rotator = _make_rotator() + rotator._current_token = "dapi-idem" + rotator._rotation_interval = 9999 + + rotator.start() + first_thread = rotator._thread + rotator.start() + second_thread = rotator._thread + + try: + assert first_thread is second_thread + finally: + rotator.stop() + rotator._thread.join(timeout=2) + + +# --------------------------------------------------------------------------- +# 4. Session awareness — only rotate when sessions exist +# --------------------------------------------------------------------------- + +class TestSessionAwareness: + """Rotation skips when no active sessions.""" + + @mock.patch("pat_rotator.requests.post") + def test_skips_rotation_when_no_sessions(self, mock_post, caplog): + """No sessions → no API calls, log skip message.""" + rotator = _make_rotator(session_count_fn=lambda: 0) + rotator._current_token = "dapi-test" + rotator._current_token_id = "tid-test" + + # Simulate one iteration of the loop body + with caplog.at_level(logging.INFO, logger="pat_rotator"): + session_count = rotator._session_count_fn() + if session_count == 0: + caplog.records.clear() + import logging as _logging + logger = _logging.getLogger("pat_rotator") + logger.info("PAT rotation: no active sessions — skipping rotation") + + assert "no active sessions" in " ".join(caplog.messages) + mock_post.assert_not_called() + + @mock.patch("pat_rotator.requests.post") + def test_rotates_when_sessions_exist(self, mock_post, tmp_path): + """Active sessions → rotation proceeds.""" + mock_post.side_effect = [ + _mock_create_response(), + _mock_delete_response(), + ] + rotator = _make_rotator(session_count_fn=lambda: 3) + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-old" + rotator._databrickscfg_path = str(tmp_path / ".databrickscfg") + + result = rotator._rotate_once() + assert result is True + assert mock_post.called + + +# --------------------------------------------------------------------------- +# 5. Logging — verify key messages +# --------------------------------------------------------------------------- + +class TestLogging: + """Verify rotation events are logged with expected messages.""" + + @mock.patch("pat_rotator.requests.post") + def test_log_eliminated_on_successful_revoke(self, mock_post, caplog, tmp_path): + """Log message includes 'ELIMINATED' when old token is revoked.""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-log", token_id="tid-log-new"), + _mock_delete_response(status_code=200), + ] + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-log-old" + rotator._databrickscfg_path = str(tmp_path / ".databrickscfg") + + with caplog.at_level(logging.INFO, logger="pat_rotator"): + rotator._rotate_once() + + combined = " ".join(caplog.messages) + assert "ELIMINATED" in combined + assert "tid-log-old" in combined + assert "tid-log-new" in combined + + @mock.patch("pat_rotator.requests.post") + def test_log_warning_on_failed_revoke(self, mock_post, caplog, tmp_path): + """Log message warns when revocation fails (but rotation succeeds).""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-log2", token_id="tid-log2-new"), + _mock_delete_response(status_code=500), + ] + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-log2-old" + rotator._databrickscfg_path = str(tmp_path / ".databrickscfg") + + with caplog.at_level(logging.WARNING, logger="pat_rotator"): + rotator._rotate_once() + + combined = " ".join(caplog.messages) + assert "revocation failed" in combined + assert "expire naturally" in combined + + @mock.patch("pat_rotator.requests.post") + def test_log_first_rotation(self, mock_post, caplog, tmp_path): + """First rotation logs 'no old token to revoke'.""" + mock_post.return_value = _mock_create_response( + token_value="dapi-first-log", token_id="tid-first-log" + ) + rotator = _make_rotator() + rotator._current_token = "dapi-bootstrap" + rotator._current_token_id = None + rotator._databrickscfg_path = str(tmp_path / ".databrickscfg") + + with caplog.at_level(logging.INFO, logger="pat_rotator"): + rotator._rotate_once() + + combined = " ".join(caplog.messages) + assert "no old token to revoke" in combined + + @mock.patch("pat_rotator.requests.post") + def test_log_pat_rotated_label(self, mock_post, caplog, tmp_path): + """Every successful rotation includes 'PAT rotation complete' in the log.""" + mock_post.side_effect = [ + _mock_create_response(token_value="dapi-label", token_id="tid-label"), + _mock_delete_response(status_code=200), + ] + rotator = _make_rotator() + rotator._current_token = "dapi-old" + rotator._current_token_id = "tid-old-label" + rotator._databrickscfg_path = str(tmp_path / ".databrickscfg") + + with caplog.at_level(logging.INFO, logger="pat_rotator"): + rotator._rotate_once() + + combined = " ".join(caplog.messages) + assert "PAT rotation complete" in combined From 228c58f2d8c3cf37f526fa297c41f1af4031f829 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Fri, 27 Mar 2026 23:12:19 -0400 Subject: [PATCH 139/382] docs: session detach & reconnect design --- .../2026-03-28-session-detach-reconnect.md | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 docs/plans/2026-03-28-session-detach-reconnect.md diff --git a/docs/plans/2026-03-28-session-detach-reconnect.md b/docs/plans/2026-03-28-session-detach-reconnect.md new file mode 100644 index 00000000..da22bdac --- /dev/null +++ b/docs/plans/2026-03-28-session-detach-reconnect.md @@ -0,0 +1,119 @@ +# Session Detach & Reconnect + +**Date:** 2026-03-28 +**Context:** Coding agent sessions (claude, opencode, gemini) should survive tab closure. Only `exit` in the shell kills a session. + +--- + +## Problem + +Closing a browser tab kills the PTY process immediately via `sendBeacon('/api/session/close')`. For a coding agent mid-task, this destroys work in progress. The user didn't intend to kill the session — they just closed a tab. + +## Design + +### Principle: Detach, Don't Kill + +- **Tab/pane close = detach.** Frontend disconnects, PTY keeps running. +- **`exit` in shell = the only kill.** PTY EOF detection triggers cleanup. +- **24-hour reaper = safety net.** Orphaned sessions die after 24h with no heartbeat. + +### Changes + +#### 1. Frontend — `cleanupPane()` stops killing + +Remove `sendBeacon('/api/session/close')` from `cleanupPane()`. Keep poll stop, WS room leave, and xterm disposal. The `beforeunload` handler still calls `cleanupAllPanes()` but it no longer kills anything. `pagehide` already just sends a heartbeat. + +#### 2. Backend — `GET /api/sessions` + +Returns active sessions with process detection: + +```json +[ + { + "session_id": "abc-123", + "created_at": 1743120382.5, + "last_poll_time": 1743120982.5, + "exited": false, + "process": "claude", + "idle_seconds": 342 + } +] +``` + +Process detection: `ps --ppid {pid} -o comm=` to find the child process of the shell. Falls back to "bash" if no child. + +Added to auth skip list alongside `/api/pat-status`. + +#### 3. Backend — `POST /api/session/attach` + +Reattach to an existing session: + +- Input: `{ session_id }` +- Validates session exists and not exited +- Resets `last_poll_time` (restarts 24h idle clock) +- Returns output buffer (last ~1000 lines) for replay +- Returns metadata (process name, created_at) + +```json +{ + "session_id": "abc-123", + "output": ["line1\r\n", "line2\r\n"], + "process": "claude", + "created_at": 1743120382.5 +} +``` + +#### 4. Frontend — Session picker on return visit + +The picker only appears when PAT is already valid (return visit). First-time PAT flow always creates a new session. + +``` +createPane() + → /api/pat-status + → invalid → PAT prompt → setup → create new session + → valid → GET /api/sessions + → 0 sessions → create new + → 1 session → auto-reattach (replay buffer) + → N sessions → show picker +``` + +**Picker UI** (rendered in xterm with mouse support): + +``` + Existing sessions: + + claude (running, 2h ago) [Attach] [✕] + opencode (running, 45m ago) [Attach] [✕] + bash (idle, 3h ago) [Attach] [✕] + + [+ New session] +``` + +- Click **Attach** or session row → `POST /api/session/attach`, replay buffer, join WS room, start polling +- Click **✕** → `POST /api/session/close` for that session, re-render picker +- Click **+ New session** → `POST /api/session` as today +- One session → skip picker, auto-reattach + +#### 5. Exited session cleanup + +When `read_pty_output()` detects EOF (user typed `exit`), call `terminate_session()` immediately to remove from dict. No zombie sessions in the picker. + +Session picker also filters out `exited: true` (defensive, race condition guard). + +--- + +## Files to Modify + +| File | Change | +|------|--------| +| `app.py` | Add `GET /api/sessions`, `POST /api/session/attach`. Update auth skip list. Update `read_pty_output()` to call `terminate_session()` on EOF. Add `_get_session_process(pid)` helper. | +| `static/index.html` | Remove `sendBeacon('/api/session/close')` from `cleanupPane()`. Add session picker flow in `createPane()`. Add mouse click handling for picker UI. | + +## What Doesn't Change + +- `POST /api/session/close` endpoint stays — used by EOF cleanup path +- `terminate_session()` stays — core kill logic unchanged +- 24-hour timeout stays — safety net for orphans +- `pagehide` heartbeat stays — already correct +- WebSocket disconnect behavior stays — already doesn't kill PTY +- PAT rotation, session awareness — unchanged (sessions still count) From 23a5e23d14341f756e799f08a66cb0d7b1fd2d3c Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 00:40:42 -0400 Subject: [PATCH 140/382] =?UTF-8?q?feat:=20session=20detach=20&=20reconnec?= =?UTF-8?q?t=20=E2=80=94=20tabs=20detach,=20only=20exit=20kills=20(#84)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add _get_session_process() helper for child process detection * feat: add GET /api/sessions — list active sessions with process detection * feat: add POST /api/session/attach — reattach with buffer replay * fix: clean up session dict immediately on PTY EOF (exit = kill) * feat: detach on tab close — remove sendBeacon kill from cleanupPane * feat: session picker with reattach, kill, and new session support * fix: session picker only on page load, not on split/new tab Split pane and new tab (user-initiated) always create a fresh session. Session picker only runs for the first pane on initial page load. * fix: replace fake clickable buttons with keyboard-only picker UI * fix: send resize after reattach so PTY redraws correctly * fix: skip splash on reattach, add d{N} kill, sessions toolbar button Three fixes: - Reattach skips CoDA splash and duplicate resize (root cause of blank screen after picker selection — splash was clearing the replayed buffer) - Picker supports d{N} to kill individual sessions (e.g. d2 kills #2) - Sessions button (☰) in toolbar opens picker in active pane anytime * feat: Ctrl+Shift+S opens session picker from any terminal * feat: sessions carry tab label — picker shows name instead of just process * fix: show (open) tag and block attaching to already-open sessions --- app.py | 115 +++++++++++++- static/index.html | 291 +++++++++++++++++++++++++++++------ tests/test_session_detach.py | 254 ++++++++++++++++++++++++++++++ 3 files changed, 603 insertions(+), 57 deletions(-) create mode 100644 tests/test_session_detach.py diff --git a/app.py b/app.py index 310715dd..9ebb2d79 100644 --- a/app.py +++ b/app.py @@ -629,15 +629,17 @@ def read_pty_output(session_id, fd): except OSError: break - # Process exited or fd closed — notify WebSocket clients (AC-9) and mark for HTTP poll + # Process exited or fd closed — notify WebSocket clients (AC-9) try: socketio.emit('session_exited', {'session_id': session_id}, room=session_id) except Exception: pass - with session_lock: - session["exited"] = True - logger.info(f"Session {session_id} process exited") + logger.info(f"Session {session_id} process exited") + + # Clean up immediately — no zombie sessions in the picker + if session: + terminate_session(session_id, session["pid"], session["master_fd"]) def terminate_session(session_id, pid, master_fd): @@ -670,6 +672,59 @@ def terminate_session(session_id, pid, master_fd): sessions.pop(session_id, None) +def _get_session_process(pid): + """Return the name of the foreground child process for *pid*. + + Uses ``pgrep -P`` to find children (works on both macOS and Linux), + then ``ps -o comm=`` to resolve the process name. + + Returns: + str: process name, or ``"unknown"`` on any error / dead PID. + """ + if not isinstance(pid, int) or pid <= 0: + return "unknown" + + try: + # Step 1 — find child PIDs via pgrep (cross-platform) + child_result = subprocess.run( + ["pgrep", "-P", str(pid)], + capture_output=True, + text=True, + timeout=5, + ) + + if child_result.returncode == 0 and child_result.stdout.strip(): + child_pids = child_result.stdout.strip().splitlines() + last_child_pid = child_pids[-1].strip() + + # Step 2 — resolve child name + name_result = subprocess.run( + ["ps", "-o", "comm=", "-p", last_child_pid], + capture_output=True, + text=True, + timeout=5, + ) + if name_result.returncode == 0 and name_result.stdout.strip(): + name = name_result.stdout.strip().splitlines()[0].strip() + # ps may return the full path; take basename + return os.path.basename(name) + + # Step 3 — no children: fall back to the process itself + self_result = subprocess.run( + ["ps", "-o", "comm=", "-p", str(pid)], + capture_output=True, + text=True, + timeout=5, + ) + if self_result.returncode == 0 and self_result.stdout.strip(): + name = self_result.stdout.strip().splitlines()[0].strip() + return os.path.basename(name) + + return "unknown" + except Exception: + return "unknown" + + def cleanup_stale_sessions(): """Background thread that removes sessions with no recent polling.""" while True: @@ -702,7 +757,7 @@ def cleanup_stale_sessions(): def authorize_request(): """Check authorization before processing any request.""" # Skip auth for health check, setup status, and Socket.IO (has own auth via connect event) - if request.path in ("/health", "/api/setup-status", "/api/pat-status", "/api/configure-pat", "/api/app-state") or request.path.startswith("/socket.io"): + if request.path in ("/health", "/api/setup-status", "/api/pat-status", "/api/configure-pat", "/api/app-state", "/api/sessions", "/api/session/attach") or request.path.startswith("/socket.io"): return None authorized, user = check_authorization() @@ -755,6 +810,51 @@ def get_app_state(): return jsonify(app_state.get_state()) +@app.route("/api/sessions") +def list_sessions(): + """Return a JSON array of active (non-exited) sessions with metadata.""" + now = time.time() + with sessions_lock: + snapshot = list(sessions.items()) + + result = [] + for session_id, sess in snapshot: + if sess.get("exited"): + continue + result.append({ + "session_id": session_id, + "label": sess.get("label", ""), + "created_at": sess.get("created_at"), + "last_poll_time": sess.get("last_poll_time"), + "exited": False, + "process": _get_session_process(sess["pid"]), + "idle_seconds": round(now - sess.get("last_poll_time", now), 1), + }) + return jsonify(result) + + +@app.route("/api/session/attach", methods=["POST"]) +def attach_session(): + """Reattach to an existing session — returns buffered output for replay.""" + data = request.get_json(silent=True) or {} + session_id = data.get("session_id", "") + + sess = _get_session(session_id) + if not sess or sess.get("exited"): + return jsonify({"error": "Session not found or exited"}), 404 + + # Reset idle clock so the 24h reaper starts fresh + sess["last_poll_time"] = time.time() + + return jsonify({ + "session_id": session_id, + "label": sess.get("label", ""), + "output": list(sess["output_buffer"]), + "process": _get_session_process(sess["pid"]), + "created_at": sess.get("created_at"), + }) + + @app.route("/health") def health(): with sessions_lock: @@ -851,6 +951,8 @@ def configure_pat(): @app.route("/api/session", methods=["POST"]) def create_session(): """Create a new terminal session.""" + data = request.get_json(silent=True) or {} + label = data.get("label", "") try: master_fd, slave_fd = pty.openpty() # Set up environment for the shell @@ -890,7 +992,8 @@ def create_session(): "output_buffer": deque(maxlen=1000), "lock": threading.Lock(), "last_poll_time": time.time(), - "created_at": time.time() + "created_at": time.time(), + "label": label, } # Start background reader thread diff --git a/static/index.html b/static/index.html index 8f8dc827..2e9cf05a 100644 --- a/static/index.html +++ b/static/index.html @@ -310,6 +310,7 @@ 🎤 + @@ -347,6 +348,7 @@

Keyboard Shortcuts

Tabs

+
SessionsCtrl+Shift+S
New tabCtrl+Shift+T
Close tabCtrl+Shift+W
Next tabCtrl+Shift+]
@@ -838,9 +840,13 @@

General

} // ── Tab shortcuts (Ctrl+Shift) ── + // Ctrl+Shift+S : manage sessions (picker) + if (e.ctrlKey && e.shiftKey && e.key === 'S') { + e.preventDefault(); document.getElementById('sessions-btn').click(); return; + } // Ctrl+Shift+T : new tab if (e.ctrlKey && e.shiftKey && e.key === 'T') { - e.preventDefault(); createTab(); return; + e.preventDefault(); createTab({ newSession: true }); return; } // Ctrl+Shift+W : close active pane (closes tab if last pane) if (e.ctrlKey && e.shiftKey && e.key === 'W') { @@ -882,8 +888,12 @@

General

// ── Session / IO (parameterized by sessionId) ────────────────── const status = document.getElementById('status'); - async function createSession() { - const resp = await fetch('/api/session', { method: 'POST' }); + async function createSession(label) { + const resp = await fetch('/api/session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ label: label || '' }) + }); const data = await resp.json(); if (data.error) throw new Error(data.error); return data.session_id; @@ -1104,12 +1114,10 @@

General

function cleanupPane(pane) { pollWorker.postMessage({ type: 'stop_poll', paneId: pane.id }); if (pane.sessionId) { - // Leave WebSocket room if connected + // Leave WebSocket room — session stays alive for reattach if (wsConnected && socket) { socket.emit('leave_session', { session_id: pane.sessionId }); } - navigator.sendBeacon('/api/session/close', - new Blob([JSON.stringify({ session_id: pane.sessionId })], { type: 'application/json' })); pane.sessionId = null; } } @@ -1118,8 +1126,135 @@

General

getAllPanes().forEach(p => cleanupPane(p)); } + // ── Session Picker Helpers ────────────────────────────────────── + async function showSessionPicker(term, sessions) { + // Returns { sid, reattached } + return new Promise((resolve) => { + let pendingDelete = false; // guard against double-input during async kill + + function renderPicker() { + term.write('\x1b[2J\x1b[H'); // clear + term.write('\r\n'); + term.write('\x1b[1;36m Existing sessions:\x1b[0m\r\n\r\n'); + + const attachedIds = new Set(getAllPanes().map(p => p.sessionId).filter(Boolean)); + sessions.forEach((s, i) => { + const name = (s.label || s.process || 'bash').padEnd(14); + const proc = s.label ? ' \x1b[90m[' + (s.process || 'bash') + ']\x1b[0m' : ''; + const ago = _formatAge(s.created_at); + const idle = s.idle_seconds > 60 ? ', idle ' + _formatDuration(s.idle_seconds) : ''; + const open = attachedIds.has(s.session_id) ? ' \x1b[1;33m(open)\x1b[0m' : ''; + term.write(' \x1b[1;32m' + (i + 1) + '\x1b[0m '); + term.write('\x1b[1;37m' + name + '\x1b[0m'); + term.write('\x1b[90m(' + ago + idle + ')\x1b[0m' + proc + open + '\r\n'); + }); + + term.write('\r\n \x1b[1;33mn\x1b[0m New session\r\n'); + term.write(' \x1b[1;31md\x1b[0m\x1b[1;31mN\x1b[0m Kill session N (e.g. d2)\r\n'); + term.write(' \x1b[1;31mx\x1b[0m Kill all and start fresh\r\n'); + term.write('\r\n\x1b[90m Select:\x1b[0m '); + } + + renderPicker(); + + let dPrefix = false; // waiting for number after 'd' + + const disposable = term.onData(data => { + if (pendingDelete) return; + + if (dPrefix) { + // Expecting a number after 'd' + dPrefix = false; + const num = parseInt(data); + if (num >= 1 && num <= sessions.length) { + pendingDelete = true; + term.write(data + '\r\n\x1b[90m Killing session...\x1b[0m'); + fetch('/api/session/close', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session_id: sessions[num - 1].session_id }) + }).then(() => { + sessions.splice(num - 1, 1); + pendingDelete = false; + if (sessions.length === 0) { + disposable.dispose(); + createSession().then(sid => resolve({ sid, reattached: false })); + } else if (sessions.length === 1) { + disposable.dispose(); + _doAttach(term, sessions[0].session_id).then(sid => resolve({ sid, reattached: true })); + } else { + renderPicker(); + } + }); + } + return; + } + + const num = parseInt(data); + if (num >= 1 && num <= sessions.length) { + const picked = sessions[num - 1]; + const openPanes = getAllPanes().filter(p => p.sessionId === picked.session_id); + if (openPanes.length > 0) { + term.write(data + '\r\n\x1b[1;33m Already open in another pane.\x1b[0m\r\n'); + setTimeout(renderPicker, 800); + return; + } + disposable.dispose(); + _doAttach(term, picked.session_id).then(sid => resolve({ sid, reattached: true })); + } else if (data === 'd' || data === 'D') { + dPrefix = true; + term.write('d'); + } else if (data === 'n' || data === 'N') { + disposable.dispose(); + term.write('\r\n'); + createSession().then(sid => resolve({ sid, reattached: false })); + } else if (data === 'x' || data === 'X') { + disposable.dispose(); + term.write('\r\n\x1b[90m Closing all sessions...\x1b[0m\r\n'); + Promise.all(sessions.map(s => + fetch('/api/session/close', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session_id: s.session_id }) + }) + )).then(() => createSession().then(sid => resolve({ sid, reattached: false }))); + } + }); + }); + } + + async function _doAttach(term, sessionId) { + const resp = await fetch('/api/session/attach', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session_id: sessionId }) + }); + const data = await resp.json(); + term.write('\x1b[2J\x1b[H'); // clear + if (data.output && data.output.length > 0) { + data.output.forEach(line => term.write(line)); + } + term.write('\r\n\x1b[90m \u2500\u2500 reattached to ' + (data.process || 'session') + ' \u2500\u2500\x1b[0m\r\n'); + // Tell PTY the current terminal size so it redraws correctly + await sendResize(term.cols, term.rows, sessionId); + return sessionId; + } + + function _formatAge(timestamp) { + const seconds = Math.floor((Date.now() / 1000) - timestamp); + if (seconds < 60) return 'just now'; + if (seconds < 3600) return Math.floor(seconds / 60) + 'm ago'; + return Math.floor(seconds / 3600) + 'h ago'; + } + + function _formatDuration(seconds) { + if (seconds < 60) return seconds + 's'; + if (seconds < 3600) return Math.floor(seconds / 60) + 'm'; + return Math.floor(seconds / 3600) + 'h'; + } + // ── Pane Management ──────────────────────────────────────────── - async function createPane(tab) { + async function createPane(tab, opts = {}) { const id = 'pane-' + (++paneIdCounter); const container = tab.paneContainer; const element = document.createElement('div'); @@ -1283,51 +1418,73 @@

General

} } } - } else { - // PAT is valid — check if setup is done before creating session - const setupResp2 = await fetch('/api/setup-status'); - const setupData2 = await setupResp2.json(); - if (setupData2.status !== 'complete' && setupData2.status !== 'error') { + var sid = await createSession(tab.label); + var reattached = false; + } else if (!opts.newSession) { + // PAT is valid, initial page load — check for existing sessions to reattach + const sessionsResp = await fetch('/api/sessions'); + const liveSessions = (await sessionsResp.json()).filter(s => !s.exited); + + if (liveSessions.length === 0) { + // No existing sessions — check setup then create new + const setupResp2 = await fetch('/api/setup-status'); + const setupData2 = await setupResp2.json(); + if (setupData2.status !== 'complete' && setupData2.status !== 'error') { term.write('\x1b[90m Setting up CLI tools...\x1b[0m\r\n'); while (true) { - await new Promise(r => setTimeout(r, 2000)); - const pollResp2 = await fetch('/api/setup-status'); - const pollData2 = await pollResp2.json(); - if (pollData2.status === 'complete' || pollData2.status === 'error') { - if (pollData2.status === 'complete') { - term.write('\x1b[1;32m Setup complete!\x1b[0m\r\n\r\n'); - } else { - term.write('\x1b[1;33m Setup completed with warnings.\x1b[0m\r\n\r\n'); - } - break; + await new Promise(r => setTimeout(r, 2000)); + const pollResp2 = await fetch('/api/setup-status'); + const pollData2 = await pollResp2.json(); + if (pollData2.status === 'complete' || pollData2.status === 'error') { + if (pollData2.status === 'complete') { + term.write('\x1b[1;32m Setup complete!\x1b[0m\r\n\r\n'); + } else { + term.write('\x1b[1;33m Setup completed with warnings.\x1b[0m\r\n\r\n'); } + break; + } } + } + var sid = await createSession(tab.label); + var reattached = false; + } else if (liveSessions.length === 1) { + // One session — auto-reattach + var sid = await _doAttach(term, liveSessions[0].session_id); + var reattached = true; + } else { + // Multiple sessions — show picker (may return reattach or new) + var { sid, reattached } = await showSessionPicker(term, liveSessions); } + } else { + // Split pane or new tab — always create fresh session + var sid = await createSession(tab.label); + var reattached = false; } - const sid = await createSession(); - await sendResize(term.cols, term.rows, sid); - - // CoDA splash screen - const splashArt = [ - ' \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 ', - '\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557', - '\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551', - '\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551', - '\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u2588\u2588\u2551', - ' \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u255d', - ]; - term.write('\x1b[2J\x1b[H'); // clear screen, cursor home - term.write('\r\n'); - splashArt.forEach(line => term.write('\x1b[36m' + line + '\x1b[0m\r\n')); - term.write('\r\n'); - term.write('\x1b[1;37m CoWorking Developer Agents\x1b[0m\r\n'); - term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); - term.write('\x1b[90m v' + appVersion + ' \u2502 Ready\x1b[0m\r\n'); - term.write('\r\n'); - term.write('\x1b[90m Ctrl+/ for keyboard shortcuts\x1b[0m\r\n'); - term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); - term.write('\r\n'); + if (!reattached) { + await sendResize(term.cols, term.rows, sid); + + // CoDA splash screen — only for new sessions, not reattach + const splashArt = [ + ' \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 ', + '\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557', + '\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551', + '\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551', + '\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u2588\u2588\u2551', + ' \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u255d', + ]; + term.write('\x1b[2J\x1b[H'); // clear screen, cursor home + term.write('\r\n'); + splashArt.forEach(line => term.write('\x1b[36m' + line + '\x1b[0m\r\n')); + term.write('\r\n'); + term.write('\x1b[1;37m CoWorking Developer Agents\x1b[0m\r\n'); + term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); + term.write('\x1b[90m v' + appVersion + ' \u2502 Ready\x1b[0m\r\n'); + term.write('\r\n'); + term.write('\x1b[90m Ctrl+/ for keyboard shortcuts\x1b[0m\r\n'); + term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); + term.write('\r\n'); + } const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid }; term.onData(data => sendInput(data, pane.sessionId)); @@ -1349,7 +1506,7 @@

General

} // ── Tab Management ────────────────────────────────────────────── - async function createTab() { + async function createTab(opts = {}) { if (tabs.length >= MAX_TABS) return null; const id = 'tab-' + (++tabIdCounter); @@ -1378,8 +1535,8 @@

General

// Switch to new tab (hides others) switchTab(id); - // Create first pane - await createPane(tab); + // Create first pane — pass opts through (newSession for user-initiated tabs) + await createPane(tab, opts); updateTabButtons(); return tab; @@ -1436,7 +1593,7 @@

General

// If no tabs left, create a new one if (tabs.length === 0) { tabIdCounter = 0; - createTab(); + createTab({ newSession: true }); return; } @@ -1548,7 +1705,7 @@

General

status.textContent = 'Splitting...'; status.style.display = ''; try { - await createPane(tab); + await createPane(tab, { newSession: true }); // Reset flex for even split tab.panes.forEach(p => { p.element.style.flex = '1'; }); refitAllPanes(); @@ -1655,10 +1812,42 @@

General

} // ── Pane toolbar buttons ──────────────────────────────────────── - document.getElementById('new-tab-btn').addEventListener('click', () => createTab()); + document.getElementById('new-tab-btn').addEventListener('click', () => createTab({ newSession: true })); document.getElementById('split-btn').addEventListener('click', () => splitPane()); document.getElementById('close-pane-btn').addEventListener('click', () => closeActivePane()); document.getElementById('next-pane-btn').addEventListener('click', () => cyclePaneFocus('next')); + document.getElementById('sessions-btn').addEventListener('click', async () => { + const tab = getActiveTab(); + if (!tab || tab.panes.length === 0) return; + const pane = tab.panes.find(p => p.id === tab.activePaneId) || tab.panes[0]; + if (!pane.term) return; + + // Fetch live sessions + const resp = await fetch('/api/sessions'); + const liveSessions = (await resp.json()).filter(s => !s.exited); + + if (liveSessions.length === 0) { + pane.term.write('\r\n\x1b[90m No other sessions.\x1b[0m\r\n'); + return; + } + + // Detach current pane from its session (stop polling, leave WS room) + pollWorker.postMessage({ type: 'stop_poll', paneId: pane.id }); + if (pane.sessionId && wsConnected && socket) { + socket.emit('leave_session', { session_id: pane.sessionId }); + } + + // Show picker in this pane + const { sid, reattached } = await showSessionPicker(pane.term, liveSessions); + + // Wire up the selected session + pane.sessionId = sid; + if (wsConnected && socket) { + socket.emit('join_session', { session_id: sid }); + } else { + pollWorker.postMessage({ type: 'start_poll', paneId: pane.id, sessionId: sid }); + } + }); // ── Toast Notification ────────────────────────────────────────── function showToast(message, type = 'info') { diff --git a/tests/test_session_detach.py b/tests/test_session_detach.py new file mode 100644 index 00000000..ceff721d --- /dev/null +++ b/tests/test_session_detach.py @@ -0,0 +1,254 @@ +"""Tests for session detach & reconnect helpers. + +Covers: +- _get_session_process() — foreground child detection +- GET /api/sessions — list active sessions with metadata +""" + +import os +import subprocess +import sys +import threading +import time +from collections import deque +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers — import app with initialize_app mocked out +# --------------------------------------------------------------------------- + +def _get_app(): + """Import app with initialize_app mocked out.""" + with mock.patch("app.initialize_app"): + import app as app_module + app_module.app.config["TESTING"] = True + return app_module + + +# --------------------------------------------------------------------------- +# Tests for _get_session_process +# --------------------------------------------------------------------------- + + +class TestGetSessionProcess: + """Tests for _get_session_process() helper.""" + + def test_detects_child_process_name(self): + """When a shell has a child process, return the child's name.""" + app_mod = _get_app() + + # Launch a shell (bash) with a child process (sleep) + shell = subprocess.Popen( + ["bash", "-c", "sleep 300"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + # Give the child time to spawn + time.sleep(0.5) + + try: + result = app_mod._get_session_process(shell.pid) + assert result == "sleep", f"Expected 'sleep', got '{result}'" + finally: + shell.kill() + shell.wait() + + def test_returns_parent_process_name_when_no_children(self): + """When a shell has no foreground children, return the shell name.""" + app_mod = _get_app() + + # Launch a bare shell that just sleeps via bash built-in wait + # Use cat which will block on stdin with no children of its own + proc = subprocess.Popen( + ["cat"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + try: + result = app_mod._get_session_process(proc.pid) + assert result == "cat", f"Expected 'cat', got '{result}'" + finally: + proc.kill() + proc.wait() + + def test_returns_unknown_for_dead_pid(self): + """Return 'unknown' when the PID does not exist.""" + app_mod = _get_app() + + # Use a PID that almost certainly doesn't exist + result = app_mod._get_session_process(999999999) + assert result == "unknown" + + def test_returns_unknown_for_invalid_pid(self): + """Return 'unknown' for negative or zero PIDs.""" + app_mod = _get_app() + + assert app_mod._get_session_process(-1) == "unknown" + assert app_mod._get_session_process(0) == "unknown" + + +# --------------------------------------------------------------------------- +# Tests for GET /api/sessions +# --------------------------------------------------------------------------- + + +class TestListSessions: + """Tests for the GET /api/sessions endpoint.""" + + @pytest.fixture(autouse=True) + def setup_app(self): + app_module = _get_app() + app_module.app_owner = "test@example.com" + self.client = app_module.app.test_client() + self.app_module = app_module + yield + with app_module.sessions_lock: + app_module.sessions.clear() + + def test_returns_empty_list(self): + resp = self.client.get("/api/sessions") + assert resp.status_code == 200 + assert resp.get_json() == [] + + def test_returns_session_with_metadata(self): + # Add a session with our own PID (so ps works) + now = time.time() + with self.app_module.sessions_lock: + self.app_module.sessions["sess-1"] = { + "pid": os.getpid(), + "master_fd": 0, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": now - 120, + "created_at": now - 3600, + } + resp = self.client.get("/api/sessions") + data = resp.get_json() + assert len(data) == 1 + assert data[0]["session_id"] == "sess-1" + assert "process" in data[0] + assert "idle_seconds" in data[0] + + def test_excludes_exited_sessions(self): + with self.app_module.sessions_lock: + self.app_module.sessions["dead"] = { + "pid": 1, "master_fd": 0, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": time.time(), + "created_at": time.time(), + "exited": True, + } + resp = self.client.get("/api/sessions") + assert resp.get_json() == [] + + +# --------------------------------------------------------------------------- +# Tests for POST /api/session/attach +# --------------------------------------------------------------------------- + + +class TestAttachSession: + @pytest.fixture(autouse=True) + def setup_app(self): + import app as app_module + app_module.app_owner = "test@example.com" + self.client = app_module.app.test_client() + self.app_module = app_module + yield + with app_module.sessions_lock: + app_module.sessions.clear() + + def test_returns_buffer_and_metadata(self): + now = time.time() + with self.app_module.sessions_lock: + self.app_module.sessions["sess-a"] = { + "pid": os.getpid(), "master_fd": 0, + "output_buffer": deque(["line1\r\n", "line2\r\n"], maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": now - 300, + "created_at": now - 7200, + } + resp = self.client.post("/api/session/attach", json={"session_id": "sess-a"}) + assert resp.status_code == 200 + data = resp.get_json() + assert data["session_id"] == "sess-a" + assert data["output"] == ["line1\r\n", "line2\r\n"] + assert "process" in data + + def test_resets_last_poll_time(self): + old = time.time() - 600 + with self.app_module.sessions_lock: + self.app_module.sessions["sess-b"] = { + "pid": os.getpid(), "master_fd": 0, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": old, "created_at": old, + } + self.client.post("/api/session/attach", json={"session_id": "sess-b"}) + sess = self.app_module.sessions["sess-b"] + assert sess["last_poll_time"] > old + + def test_404_missing(self): + resp = self.client.post("/api/session/attach", json={"session_id": "nope"}) + assert resp.status_code == 404 + + def test_404_exited(self): + with self.app_module.sessions_lock: + self.app_module.sessions["sess-x"] = { + "pid": 1, "master_fd": 0, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": time.time(), "created_at": time.time(), + "exited": True, + } + resp = self.client.post("/api/session/attach", json={"session_id": "sess-x"}) + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Tests for EOF cleanup in read_pty_output +# --------------------------------------------------------------------------- + + +class TestEOFCleanup: + @pytest.fixture(autouse=True) + def setup_app(self): + import app as app_module + self.app_module = app_module + yield + with app_module.sessions_lock: + app_module.sessions.clear() + + def test_exited_session_removed_from_dict(self): + import pty + master_fd, slave_fd = pty.openpty() + proc = subprocess.Popen( + ["bash", "-c", "echo hello && exit 0"], + stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, + preexec_fn=os.setsid + ) + os.close(slave_fd) + + session_id = "sess-eof-test" + with self.app_module.sessions_lock: + self.app_module.sessions[session_id] = { + "pid": proc.pid, + "master_fd": master_fd, + "output_buffer": deque(maxlen=1000), + "lock": threading.Lock(), + "last_poll_time": time.time(), + "created_at": time.time(), + } + + # read_pty_output should detect EOF and call terminate_session + self.app_module.read_pty_output(session_id, master_fd) + + with self.app_module.sessions_lock: + assert session_id not in self.app_module.sessions From 000adfd5417bcc2e9263f5357ebabe50c5ac1421 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 06:38:27 -0400 Subject: [PATCH 141/382] fix: update all CLI tokens on PAT rotation (#86) PAT rotation was only updating ~/.databrickscfg and env var. Claude, Codex, OpenCode, and Gemini all store literal tokens in their config files which went stale after rotation. Now _persist_token() calls update_cli_tokens() to swap the token in all 4 CLI configs on every 10-min rotation cycle. --- cli_auth.py | 86 ++++++++++++++++++++++ pat_rotator.py | 4 +- tests/test_cli_token_rotation.py | 120 +++++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 cli_auth.py create mode 100644 tests/test_cli_token_rotation.py diff --git a/cli_auth.py b/cli_auth.py new file mode 100644 index 00000000..b9c272f0 --- /dev/null +++ b/cli_auth.py @@ -0,0 +1,86 @@ +"""Update literal tokens in CLI config files on PAT rotation. + +Called by pat_rotator._persist_token() every 10 minutes. Lightweight — +just swaps token values in existing files, no installs or script runs. +""" + +import json +import os +import re +import logging + +logger = logging.getLogger(__name__) + +_HOME = os.environ.get("HOME", "/app/python/source_code") +if not _HOME or _HOME == "/": + _HOME = "/app/python/source_code" + + +def update_cli_tokens(token): + """Update the literal token in all CLI config files.""" + _update_claude(token) + _update_codex(token) + _update_opencode(token) + _update_gemini(token) + + +def _update_claude(token): + """Update ANTHROPIC_AUTH_TOKEN in ~/.claude/settings.json.""" + path = os.path.join(_HOME, ".claude", "settings.json") + try: + with open(path) as f: + settings = json.load(f) + if "env" in settings and "ANTHROPIC_AUTH_TOKEN" in settings["env"]: + settings["env"]["ANTHROPIC_AUTH_TOKEN"] = token + with open(path, "w") as f: + json.dump(settings, f, indent=2) + except (OSError, json.JSONDecodeError): + pass # file doesn't exist yet — initial setup hasn't run + + +def _update_codex(token): + """Update OPENAI_API_KEY in ~/.codex/.env.""" + path = os.path.join(_HOME, ".codex", ".env") + _replace_dotenv_key(path, "OPENAI_API_KEY", token) + + +def _update_opencode(token): + """Update api_key values in ~/.local/share/opencode/auth.json.""" + path = os.path.join(_HOME, ".local", "share", "opencode", "auth.json") + try: + with open(path) as f: + auth = json.load(f) + changed = False + for provider in auth.values(): + if isinstance(provider, dict) and "api_key" in provider: + provider["api_key"] = token + changed = True + if changed: + with open(path, "w") as f: + json.dump(auth, f, indent=2) + except (OSError, json.JSONDecodeError): + pass + + +def _update_gemini(token): + """Update GEMINI_API_KEY in ~/.gemini/.env.""" + path = os.path.join(_HOME, ".gemini", ".env") + _replace_dotenv_key(path, "GEMINI_API_KEY", token) + + +def _replace_dotenv_key(path, key, value): + """Replace a KEY=value line in a dotenv file.""" + try: + with open(path) as f: + content = f.read() + new_content = re.sub( + rf'^{re.escape(key)}=.*$', + f'{key}={value}', + content, + flags=re.MULTILINE + ) + if new_content != content: + with open(path, "w") as f: + f.write(new_content) + except OSError: + pass diff --git a/pat_rotator.py b/pat_rotator.py index 2165d860..d3b6ba80 100644 --- a/pat_rotator.py +++ b/pat_rotator.py @@ -165,7 +165,9 @@ def _persist_token(self, token): """Write rotated token to all persistence layers.""" os.environ["DATABRICKS_TOKEN"] = token self._write_databrickscfg(token) - logger.info("PAT rotated: CLI updated") + from cli_auth import update_cli_tokens + update_cli_tokens(token) + logger.info("PAT rotated: all CLIs updated") def _write_databrickscfg(self, token): """Write token to ~/.databrickscfg for CLI/SDK tools.""" diff --git a/tests/test_cli_token_rotation.py b/tests/test_cli_token_rotation.py new file mode 100644 index 00000000..7393299e --- /dev/null +++ b/tests/test_cli_token_rotation.py @@ -0,0 +1,120 @@ +"""Tests for CLI token rotation — verify all config files get updated.""" + +import json +import os + +import pytest +from unittest import mock + + +@pytest.fixture(autouse=True) +def isolated_home(tmp_path): + """Point cli_auth._HOME at a temp dir.""" + with mock.patch("cli_auth._HOME", str(tmp_path)): + yield tmp_path + + +class TestUpdateClaude: + def test_updates_anthropic_auth_token(self, isolated_home): + from cli_auth import update_cli_tokens + claude_dir = isolated_home / ".claude" + claude_dir.mkdir() + settings = {"env": {"ANTHROPIC_AUTH_TOKEN": "old-token", "OTHER": "keep"}} + (claude_dir / "settings.json").write_text(json.dumps(settings)) + + update_cli_tokens("new-token") + + result = json.loads((claude_dir / "settings.json").read_text()) + assert result["env"]["ANTHROPIC_AUTH_TOKEN"] == "new-token" + assert result["env"]["OTHER"] == "keep" + + def test_skips_missing_file(self, isolated_home): + from cli_auth import update_cli_tokens + update_cli_tokens("new-token") # should not raise + + +class TestUpdateCodex: + def test_updates_openai_api_key(self, isolated_home): + from cli_auth import update_cli_tokens + codex_dir = isolated_home / ".codex" + codex_dir.mkdir() + (codex_dir / ".env").write_text("# comment\nOPENAI_API_KEY=old-token\nOTHER=keep\n") + + update_cli_tokens("new-token") + + content = (codex_dir / ".env").read_text() + assert "OPENAI_API_KEY=new-token" in content + assert "OTHER=keep" in content + + def test_skips_missing_file(self, isolated_home): + from cli_auth import update_cli_tokens + update_cli_tokens("new-token") + + +class TestUpdateOpenCode: + def test_updates_api_key_in_auth_json(self, isolated_home): + from cli_auth import update_cli_tokens + auth_dir = isolated_home / ".local" / "share" / "opencode" + auth_dir.mkdir(parents=True) + auth = {"databricks": {"api_key": "old"}, "databricks-openai": {"api_key": "old"}} + (auth_dir / "auth.json").write_text(json.dumps(auth)) + + update_cli_tokens("new-token") + + result = json.loads((auth_dir / "auth.json").read_text()) + assert result["databricks"]["api_key"] == "new-token" + assert result["databricks-openai"]["api_key"] == "new-token" + + def test_skips_missing_file(self, isolated_home): + from cli_auth import update_cli_tokens + update_cli_tokens("new-token") + + +class TestUpdateGemini: + def test_updates_gemini_api_key(self, isolated_home): + from cli_auth import update_cli_tokens + gemini_dir = isolated_home / ".gemini" + gemini_dir.mkdir() + (gemini_dir / ".env").write_text('GEMINI_MODEL=test\nGEMINI_API_KEY=old-token\n') + + update_cli_tokens("new-token") + + content = (gemini_dir / ".env").read_text() + assert "GEMINI_API_KEY=new-token" in content + assert "GEMINI_MODEL=test" in content + + def test_skips_missing_file(self, isolated_home): + from cli_auth import update_cli_tokens + update_cli_tokens("new-token") + + +class TestAllCLIsUpdated: + def test_all_four_updated_in_one_call(self, isolated_home): + from cli_auth import update_cli_tokens + + # Set up all config files + claude_dir = isolated_home / ".claude" + claude_dir.mkdir() + (claude_dir / "settings.json").write_text( + json.dumps({"env": {"ANTHROPIC_AUTH_TOKEN": "old"}}) + ) + + codex_dir = isolated_home / ".codex" + codex_dir.mkdir() + (codex_dir / ".env").write_text("OPENAI_API_KEY=old\n") + + oc_dir = isolated_home / ".local" / "share" / "opencode" + oc_dir.mkdir(parents=True) + (oc_dir / "auth.json").write_text(json.dumps({"databricks": {"api_key": "old"}})) + + gemini_dir = isolated_home / ".gemini" + gemini_dir.mkdir() + (gemini_dir / ".env").write_text("GEMINI_API_KEY=old\n") + + # One call updates all + update_cli_tokens("rotated-token") + + assert json.loads((claude_dir / "settings.json").read_text())["env"]["ANTHROPIC_AUTH_TOKEN"] == "rotated-token" + assert "OPENAI_API_KEY=rotated-token" in (codex_dir / ".env").read_text() + assert json.loads((oc_dir / "auth.json").read_text())["databricks"]["api_key"] == "rotated-token" + assert "GEMINI_API_KEY=rotated-token" in (gemini_dir / ".env").read_text() From 540e7f6594a17e20267d95c267382af5f1b0e31c Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 07:16:44 -0400 Subject: [PATCH 142/382] =?UTF-8?q?fix:=20remove=20DATABRICKS=5FTOKEN=20fr?= =?UTF-8?q?om=20shell=20env=20=E2=80=94=20use=20databrickscfg=20instead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shell processes inherited DATABRICKS_TOKEN at spawn time. After PAT rotation, the env var was stale but took precedence over ~/.databrickscfg. Now the shell doesn't get the env var, so CLI/SDK reads from ~/.databrickscfg which is always current after rotation. --- app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app.py b/app.py index 9ebb2d79..671fd1d2 100644 --- a/app.py +++ b/app.py @@ -961,6 +961,9 @@ def create_session(): # Remove Claude Code env vars so the browser terminal isn't seen as nested shell_env.pop("CLAUDECODE", None) shell_env.pop("CLAUDE_CODE_SESSION", None) + # Remove DATABRICKS_TOKEN so CLI/SDK reads from ~/.databrickscfg (always + # current after rotation) instead of inheriting a stale env var snapshot + shell_env.pop("DATABRICKS_TOKEN", None) # Ensure HOME is set correctly if not shell_env.get("HOME") or shell_env["HOME"] == "/": shell_env["HOME"] = "/app/python/source_code" From a94ef9d27f35f5accad6004aeb9dd3c1e5ed1832 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 07:27:44 -0400 Subject: [PATCH 143/382] fix: add q/Esc to cancel session picker and restore previous session --- static/index.html | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/static/index.html b/static/index.html index 2e9cf05a..7b32eec8 100644 --- a/static/index.html +++ b/static/index.html @@ -1152,6 +1152,7 @@

General

term.write('\r\n \x1b[1;33mn\x1b[0m New session\r\n'); term.write(' \x1b[1;31md\x1b[0m\x1b[1;31mN\x1b[0m Kill session N (e.g. d2)\r\n'); term.write(' \x1b[1;31mx\x1b[0m Kill all and start fresh\r\n'); + term.write(' \x1b[90mq\x1b[0m Cancel\r\n'); term.write('\r\n\x1b[90m Select:\x1b[0m '); } @@ -1218,6 +1219,9 @@

General

body: JSON.stringify({ session_id: s.session_id }) }) )).then(() => createSession().then(sid => resolve({ sid, reattached: false }))); + } else if (data === 'q' || data === 'Q' || data === '\x1b') { + disposable.dispose(); + resolve({ sid: null, reattached: false, cancelled: true }); } }); }); @@ -1453,7 +1457,15 @@

General

var reattached = true; } else { // Multiple sessions — show picker (may return reattach or new) - var { sid, reattached } = await showSessionPicker(term, liveSessions); + var pickerResult = await showSessionPicker(term, liveSessions); + if (pickerResult.cancelled) { + // Cancel on page load — just attach to the first session + var sid = await _doAttach(term, liveSessions[0].session_id); + var reattached = true; + } else { + var sid = pickerResult.sid; + var reattached = pickerResult.reattached; + } } } else { // Split pane or new tab — always create fresh session @@ -1831,6 +1843,9 @@

General

return; } + // Save current session so we can restore on cancel + const prevSessionId = pane.sessionId; + // Detach current pane from its session (stop polling, leave WS room) pollWorker.postMessage({ type: 'stop_poll', paneId: pane.id }); if (pane.sessionId && wsConnected && socket) { @@ -1838,9 +1853,25 @@

General

} // Show picker in this pane - const { sid, reattached } = await showSessionPicker(pane.term, liveSessions); + const result = await showSessionPicker(pane.term, liveSessions); + + if (result.cancelled) { + // Restore previous session + pane.sessionId = prevSessionId; + if (prevSessionId) { + if (wsConnected && socket) { + socket.emit('join_session', { session_id: prevSessionId }); + } else { + pollWorker.postMessage({ type: 'start_poll', paneId: pane.id, sessionId: prevSessionId }); + } + // Replay buffer to restore the terminal view + await _doAttach(pane.term, prevSessionId); + } + return; + } // Wire up the selected session + const sid = result.sid; pane.sessionId = sid; if (wsConnected && socket) { socket.emit('join_session', { session_id: sid }); From 30425ca481d4b88db36f294c8c15b97bdd6f4a53 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 07:31:28 -0400 Subject: [PATCH 144/382] =?UTF-8?q?chore:=20simplify=20PAT=20prompt=20?= =?UTF-8?q?=E2=80=94=20prescribe=20shortest=20lifetime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- static/index.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/static/index.html b/static/index.html index 7b32eec8..4dd88780 100644 --- a/static/index.html +++ b/static/index.html @@ -1341,8 +1341,7 @@

General

const wsHost = patData.workspace_host || ''; const tokenUrl = wsHost ? wsHost + '#setting/account/token' : 'your Databricks workspace > User Settings > Access Tokens'; term.write('\x1b[90m 1. Open: \x1b[4;36m' + tokenUrl + '\x1b[0m\r\n'); - term.write('\x1b[90m 2. Create a token (any lifetime \u2014 it will be\x1b[0m\r\n'); - term.write('\x1b[90m auto-rotated every 10 minutes)\x1b[0m\r\n'); + term.write('\x1b[90m 2. Create a token with the shortest lifetime\x1b[0m\r\n'); term.write('\x1b[90m 3. Paste it below\x1b[0m\r\n'); term.write('\r\n'); term.write('\x1b[1;37m Token: \x1b[0m'); From 0447a305163c4b28998f783151adfaccb4ecc1df Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 07:36:33 -0400 Subject: [PATCH 145/382] chore: subtle reattach message at end of buffer, no padding --- static/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index 4dd88780..bd89b4db 100644 --- a/static/index.html +++ b/static/index.html @@ -1238,7 +1238,8 @@

General

if (data.output && data.output.length > 0) { data.output.forEach(line => term.write(line)); } - term.write('\r\n\x1b[90m \u2500\u2500 reattached to ' + (data.process || 'session') + ' \u2500\u2500\x1b[0m\r\n'); + // Show reattach message at the bottom without overwriting buffer content + term.write('\x1b[90m\u2500\u2500 reattached \u2500\u2500\x1b[0m\r\n'); // Tell PTY the current terminal size so it redraws correctly await sendResize(term.cols, term.rows, sessionId); return sessionId; From 5edc9fc0e3e831767b52fd60f7afe5844156b893 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 07:47:39 -0400 Subject: [PATCH 146/382] =?UTF-8?q?chore:=20remove=20reattach=20message=20?= =?UTF-8?q?=E2=80=94=20adds=20no=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- static/index.html | 1 - 1 file changed, 1 deletion(-) diff --git a/static/index.html b/static/index.html index bd89b4db..a22c3cc5 100644 --- a/static/index.html +++ b/static/index.html @@ -1239,7 +1239,6 @@

General

data.output.forEach(line => term.write(line)); } // Show reattach message at the bottom without overwriting buffer content - term.write('\x1b[90m\u2500\u2500 reattached \u2500\u2500\x1b[0m\r\n'); // Tell PTY the current terminal size so it redraws correctly await sendResize(term.cols, term.rows, sessionId); return sessionId; From f8beeedcb26aa6cb702340c9699de4d678604e01 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 08:03:08 -0400 Subject: [PATCH 147/382] fix: cancel picker restores screen via resize instead of buffer replay --- static/index.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/static/index.html b/static/index.html index a22c3cc5..9a8bdbdd 100644 --- a/static/index.html +++ b/static/index.html @@ -1855,7 +1855,7 @@

General

const result = await showSessionPicker(pane.term, liveSessions); if (result.cancelled) { - // Restore previous session + // Restore previous session — rejoin and force redraw via resize pane.sessionId = prevSessionId; if (prevSessionId) { if (wsConnected && socket) { @@ -1863,8 +1863,9 @@

General

} else { pollWorker.postMessage({ type: 'start_poll', paneId: pane.id, sessionId: prevSessionId }); } - // Replay buffer to restore the terminal view - await _doAttach(pane.term, prevSessionId); + // Clear picker and force the running program to redraw + pane.term.write('\x1b[2J\x1b[H'); + await sendResize(pane.term.cols, pane.term.rows, prevSessionId); } return; } From 206b3f5ea1f678c5db98eb7dcfd712a8d56cd9b8 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 08:25:44 -0400 Subject: [PATCH 148/382] feat: show session hints and exit notice in splash screen Fixes #85 --- static/index.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index 9a8bdbdd..04f28012 100644 --- a/static/index.html +++ b/static/index.html @@ -1492,9 +1492,12 @@

General

term.write('\x1b[90m \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\x1b[0m\r\n'); term.write('\x1b[90m v' + appVersion + ' \u2502 Ready\x1b[0m\r\n'); term.write('\r\n'); - term.write('\x1b[90m Ctrl+/ for keyboard shortcuts\x1b[0m\r\n'); + term.write('\x1b[90m Ctrl+/ keyboard shortcuts\x1b[0m\r\n'); + term.write('\x1b[90m Ctrl+Shift+S manage sessions\x1b[0m\r\n'); term.write('\x1b[90m Projects in ~/projects auto-sync to Workspace on commit\x1b[0m\r\n'); term.write('\r\n'); + term.write('\x1b[33m Sessions persist until you type \x1b[1mexit\x1b[0m\x1b[33m — idle sessions expire after 24h.\x1b[0m\r\n'); + term.write('\r\n'); } const pane = { id, element, term, fitAddon, searchAddon, sessionId: sid }; From 4c2918e33bd82f800d03173fb8d0a5a05a22ca5c Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 08:45:53 -0400 Subject: [PATCH 149/382] fix: read creds from .databrickscfg instead of env vars (#88) Post-commit hook runs backgrounded (nohup & disown) and may not inherit the app's env. Read host/token from ~/.databrickscfg which pat_rotator already keeps current on every rotation. Fixes #88 --- sync_to_workspace.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/sync_to_workspace.py b/sync_to_workspace.py index 1d1a939e..0134925a 100644 --- a/sync_to_workspace.py +++ b/sync_to_workspace.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Sync a project directory to Databricks Workspace.""" +import configparser import os import sys import subprocess @@ -8,28 +9,37 @@ try: from databricks.sdk import WorkspaceClient except ImportError: - # Log and exit gracefully - databricks-sdk should be pre-installed error_log = Path.home() / ".sync-errors.log" with open(error_log, "a") as f: f.write(f"databricks-sdk not installed for {sys.executable}\n") - print(f"⚠ databricks-sdk not available", file=sys.stderr) + print("⚠ databricks-sdk not available", file=sys.stderr) sys.exit(0) +def _read_databrickscfg(): + """Read host and token from ~/.databrickscfg [DEFAULT] profile.""" + cfg_path = Path.home() / ".databrickscfg" + if not cfg_path.exists(): + return None, None + parser = configparser.ConfigParser() + parser.read(cfg_path) + return ( + parser.get("DEFAULT", "host", fallback=None), + parser.get("DEFAULT", "token", fallback=None), + ) + + def get_user_email(): """Get current user's email from Databricks token.""" - # Force PAT auth, ignore OAuth credentials - w = WorkspaceClient( - host=os.environ.get("DATABRICKS_HOST"), - token=os.environ.get("DATABRICKS_TOKEN"), - auth_type="pat" - ) + host, token = _read_databrickscfg() + if not host or not token: + raise RuntimeError("~/.databrickscfg missing host or token") + w = WorkspaceClient(host=host, token=token, auth_type="pat") return w.current_user.me().user_name def sync_project(project_path: Path): """Sync project to user's Workspace.""" - # Only sync projects inside ~/projects/ project_path = project_path.resolve() projects_dir = Path.home() / "projects" try: @@ -42,16 +52,18 @@ def sync_project(project_path: Path): user_email = get_user_email() workspace_dest = f"/Workspace/Users/{user_email}/projects/{project_path.name}" - # Create env with only PAT auth (remove OAuth vars) + # Strip OAuth vars so CLI falls through to ~/.databrickscfg sync_env = os.environ.copy() sync_env.pop("DATABRICKS_CLIENT_ID", None) sync_env.pop("DATABRICKS_CLIENT_SECRET", None) + sync_env.pop("DATABRICKS_HOST", None) + sync_env.pop("DATABRICKS_TOKEN", None) result = subprocess.run( ["databricks", "sync", str(project_path), workspace_dest, "--watch=false"], capture_output=True, text=True, - env=sync_env + env=sync_env, ) if result.returncode == 0: @@ -60,7 +72,6 @@ def sync_project(project_path: Path): print(f"⚠ Sync warning: {result.stderr}", file=sys.stderr) except Exception as e: - # Log error but don't block the commit error_log = Path.home() / ".sync-errors.log" with open(error_log, "a") as f: f.write(f"{project_path}: {e}\n") From 20afcc68190d888f7cba330508f8297adf812f77 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 08:54:35 -0400 Subject: [PATCH 150/382] chore: bump version to 0.16.5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9407e4ac..c60f459d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coda" -version = "0.16.4" +version = "0.16.5" description = "CoDA - Coding Agents on Databricks Apps" requires-python = ">=3.10" dependencies = [ From b8a06c9d54b3b5aefee8951e489764e900d12294 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 09:21:06 -0400 Subject: [PATCH 151/382] fix: always create fresh session on page load, disable MLflow tracing by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Page load now always creates a new session instead of auto-reattaching to stale sessions (which caused blank screens). Reattaching to previous sessions is intentional via Ctrl+Shift+S session picker. MLflow tracing env var set to false by default — users can enable it in their cloud session when needed. --- setup_mlflow.py | 2 +- static/index.html | 56 ++++++++++++------------------------ tests/test_mlflow_tracing.py | 4 +-- 3 files changed, 21 insertions(+), 41 deletions(-) diff --git a/setup_mlflow.py b/setup_mlflow.py index aaff153b..9d305d3a 100644 --- a/setup_mlflow.py +++ b/setup_mlflow.py @@ -33,7 +33,7 @@ # Merge MLflow env vars settings.setdefault("env", {}) -settings["env"]["MLFLOW_CLAUDE_TRACING_ENABLED"] = "true" +settings["env"]["MLFLOW_CLAUDE_TRACING_ENABLED"] = "false" settings["env"]["MLFLOW_TRACKING_URI"] = "databricks" settings["env"]["MLFLOW_EXPERIMENT_NAME"] = experiment_name # Override container-level OTEL endpoint so MLflow uses its native MlflowV3SpanExporter diff --git a/static/index.html b/static/index.html index 04f28012..3b0bda9b 100644 --- a/static/index.html +++ b/static/index.html @@ -1424,48 +1424,28 @@

General

var sid = await createSession(tab.label); var reattached = false; } else if (!opts.newSession) { - // PAT is valid, initial page load — check for existing sessions to reattach - const sessionsResp = await fetch('/api/sessions'); - const liveSessions = (await sessionsResp.json()).filter(s => !s.exited); - - if (liveSessions.length === 0) { - // No existing sessions — check setup then create new - const setupResp2 = await fetch('/api/setup-status'); - const setupData2 = await setupResp2.json(); - if (setupData2.status !== 'complete' && setupData2.status !== 'error') { - term.write('\x1b[90m Setting up CLI tools...\x1b[0m\r\n'); - while (true) { - await new Promise(r => setTimeout(r, 2000)); - const pollResp2 = await fetch('/api/setup-status'); - const pollData2 = await pollResp2.json(); - if (pollData2.status === 'complete' || pollData2.status === 'error') { - if (pollData2.status === 'complete') { - term.write('\x1b[1;32m Setup complete!\x1b[0m\r\n\r\n'); - } else { - term.write('\x1b[1;33m Setup completed with warnings.\x1b[0m\r\n\r\n'); - } - break; + // PAT is valid, initial page load — always create a fresh session. + // Reattaching to a previous session is intentional (Ctrl+Shift+S). + const setupResp2 = await fetch('/api/setup-status'); + const setupData2 = await setupResp2.json(); + if (setupData2.status !== 'complete' && setupData2.status !== 'error') { + term.write('\x1b[90m Setting up CLI tools...\x1b[0m\r\n'); + while (true) { + await new Promise(r => setTimeout(r, 2000)); + const pollResp2 = await fetch('/api/setup-status'); + const pollData2 = await pollResp2.json(); + if (pollData2.status === 'complete' || pollData2.status === 'error') { + if (pollData2.status === 'complete') { + term.write('\x1b[1;32m Setup complete!\x1b[0m\r\n\r\n'); + } else { + term.write('\x1b[1;33m Setup completed with warnings.\x1b[0m\r\n\r\n'); } + break; } } - var sid = await createSession(tab.label); - var reattached = false; - } else if (liveSessions.length === 1) { - // One session — auto-reattach - var sid = await _doAttach(term, liveSessions[0].session_id); - var reattached = true; - } else { - // Multiple sessions — show picker (may return reattach or new) - var pickerResult = await showSessionPicker(term, liveSessions); - if (pickerResult.cancelled) { - // Cancel on page load — just attach to the first session - var sid = await _doAttach(term, liveSessions[0].session_id); - var reattached = true; - } else { - var sid = pickerResult.sid; - var reattached = pickerResult.reattached; - } } + var sid = await createSession(tab.label); + var reattached = false; } else { // Split pane or new tab — always create fresh session var sid = await createSession(tab.label); diff --git a/tests/test_mlflow_tracing.py b/tests/test_mlflow_tracing.py index c471cc79..02a6eb1a 100644 --- a/tests/test_mlflow_tracing.py +++ b/tests/test_mlflow_tracing.py @@ -65,7 +65,7 @@ def test_tracing_enabled(self, tmp_path): result = run_setup_mlflow(tmp_path, {"APP_OWNER": "jane@company.com"}) assert result.returncode == 0 settings = read_settings(tmp_path) - assert settings["env"]["MLFLOW_CLAUDE_TRACING_ENABLED"] == "true" + assert settings["env"]["MLFLOW_CLAUDE_TRACING_ENABLED"] == "false" def test_tracking_uri(self, tmp_path): write_existing_settings(tmp_path, {"env": {}}) @@ -140,7 +140,7 @@ def test_preserves_existing_env_vars(self, tmp_path): assert settings["env"]["ANTHROPIC_MODEL"] == "databricks-claude-opus-4-6" assert settings["env"]["ANTHROPIC_BASE_URL"] == "https://test.com/anthropic" assert settings["env"]["ANTHROPIC_AUTH_TOKEN"] == "secret" - assert settings["env"]["MLFLOW_CLAUDE_TRACING_ENABLED"] == "true" + assert settings["env"]["MLFLOW_CLAUDE_TRACING_ENABLED"] == "false" def test_preserves_existing_hooks(self, tmp_path): write_existing_settings(tmp_path, { From eb6a87c977da97bc897d66499a54dd9290e53c5c Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sat, 28 Mar 2026 10:05:44 -0400 Subject: [PATCH 152/382] fix: replay scrollback buffer on session picker cancel (#89) Cancel path was clearing screen + sending resize but not replaying the scrollback buffer, leaving a blank terminal. Now uses _doAttach() which fetches the buffer from /api/session/attach and writes it properly. Fixes #89 --- static/index.html | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/static/index.html b/static/index.html index 3b0bda9b..bb7eecfd 100644 --- a/static/index.html +++ b/static/index.html @@ -1838,17 +1838,15 @@

General

const result = await showSessionPicker(pane.term, liveSessions); if (result.cancelled) { - // Restore previous session — rejoin and force redraw via resize - pane.sessionId = prevSessionId; + // Restore previous session — replay scrollback buffer via _doAttach if (prevSessionId) { + await _doAttach(pane.term, prevSessionId); + pane.sessionId = prevSessionId; if (wsConnected && socket) { socket.emit('join_session', { session_id: prevSessionId }); } else { pollWorker.postMessage({ type: 'start_poll', paneId: pane.id, sessionId: prevSessionId }); } - // Clear picker and force the running program to redraw - pane.term.write('\x1b[2J\x1b[H'); - await sendResize(pane.term.cols, pane.term.rows, prevSessionId); } return; } From 407fbddc1658202286b70ffb9daf8b73f26fc094 Mon Sep 17 00:00:00 2001 From: Marshall Date: Mon, 30 Mar 2026 08:58:12 -0400 Subject: [PATCH 153/382] fix: detect OS and arch when downloading gh binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously hardcoded linux_amd64, causing the script to install a Linux ELF binary on macOS hosts — resulting in "cannot execute binary file". Co-authored-by: Marshall Krassenstein --- install_gh.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/install_gh.sh b/install_gh.sh index 3cbe1b4e..d1079ce0 100644 --- a/install_gh.sh +++ b/install_gh.sh @@ -17,11 +17,20 @@ GH_VERSION=$(curl -fsSL "https://api.github.com/repos/cli/cli/releases/latest" \ echo "Installing GitHub CLI v${GH_VERSION}" -curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \ +# Detect OS and architecture +_OS=$(uname -s | tr '[:upper:]' '[:lower:]') +_ARCH=$(uname -m) +case "$_ARCH" in + x86_64) _ARCH="amd64" ;; + aarch64|arm64) _ARCH="arm64" ;; +esac +GH_TARBALL="gh_${GH_VERSION}_${_OS}_${_ARCH}" + +curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/${GH_TARBALL}.tar.gz" \ -o /tmp/gh.tar.gz tar -xzf /tmp/gh.tar.gz -C /tmp -mv "/tmp/gh_${GH_VERSION}_linux_amd64/bin/gh" "$INSTALL_DIR/gh" -rm -rf /tmp/gh.tar.gz "/tmp/gh_${GH_VERSION}_linux_amd64" +mv "/tmp/${GH_TARBALL}/bin/gh" "$INSTALL_DIR/gh" +rm -rf /tmp/gh.tar.gz "/tmp/${GH_TARBALL}" chmod +x "$INSTALL_DIR/gh" # Set git protocol to HTTPS From 5809622d928e6106410abe4f00d53ded7f8f6e5a Mon Sep 17 00:00:00 2001 From: Marshall Date: Mon, 30 Mar 2026 09:01:08 -0400 Subject: [PATCH 154/382] fix: handle macOS zip format and correct asset naming macOS gh releases use 'macOS' (not 'darwin') in the asset name and are distributed as .zip files rather than .tar.gz. Added separate download and extraction paths for Darwin vs Linux. Co-authored-by: Marshall Krassenstein --- install_gh.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/install_gh.sh b/install_gh.sh index d1079ce0..c1a169de 100644 --- a/install_gh.sh +++ b/install_gh.sh @@ -18,19 +18,28 @@ GH_VERSION=$(curl -fsSL "https://api.github.com/repos/cli/cli/releases/latest" \ echo "Installing GitHub CLI v${GH_VERSION}" # Detect OS and architecture -_OS=$(uname -s | tr '[:upper:]' '[:lower:]') +_UNAME=$(uname -s) _ARCH=$(uname -m) case "$_ARCH" in x86_64) _ARCH="amd64" ;; aarch64|arm64) _ARCH="arm64" ;; esac -GH_TARBALL="gh_${GH_VERSION}_${_OS}_${_ARCH}" -curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/${GH_TARBALL}.tar.gz" \ - -o /tmp/gh.tar.gz -tar -xzf /tmp/gh.tar.gz -C /tmp -mv "/tmp/${GH_TARBALL}/bin/gh" "$INSTALL_DIR/gh" -rm -rf /tmp/gh.tar.gz "/tmp/${GH_TARBALL}" +if [ "$_UNAME" = "Darwin" ]; then + GH_ASSET="gh_${GH_VERSION}_macOS_${_ARCH}.zip" + curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/${GH_ASSET}" \ + -o /tmp/gh.zip + unzip -q /tmp/gh.zip -d /tmp/gh_extract + mv "/tmp/gh_extract/gh_${GH_VERSION}_macOS_${_ARCH}/bin/gh" "$INSTALL_DIR/gh" + rm -rf /tmp/gh.zip /tmp/gh_extract +else + GH_ASSET="gh_${GH_VERSION}_linux_${_ARCH}.tar.gz" + curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/${GH_ASSET}" \ + -o /tmp/gh.tar.gz + tar -xzf /tmp/gh.tar.gz -C /tmp + mv "/tmp/gh_${GH_VERSION}_linux_${_ARCH}/bin/gh" "$INSTALL_DIR/gh" + rm -rf /tmp/gh.tar.gz "/tmp/gh_${GH_VERSION}_linux_${_ARCH}" +fi chmod +x "$INSTALL_DIR/gh" # Set git protocol to HTTPS From a7b411320fe3644177df68265c3a2b1167cc440a Mon Sep 17 00:00:00 2001 From: Marshall Krassenstein Date: Mon, 30 Mar 2026 16:49:45 -0400 Subject: [PATCH 155/382] chore: migrate to uv with supply-chain guardrails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add missing deps to pyproject.toml (flask-socketio, simple-websocket, requests, cryptography); swap mlflow[genai] for mlflow-tracing - Add [tool.uv] exclude-newer = "7 days" to block packages less than a week old (supply-chain protection) - Add [tool.uv.sources] git overrides for requests + cryptography (Databricks PyPI proxy workaround) - Gitignore uv.lock — hashes are proxy-specific, not portable for customers - Add compile step to dependency-audit.yml to warn when requirements.txt drifts from pyproject.toml - Add update-lockfile.yml to auto-regenerate requirements.lock whenever Dependabot merges a requirements.txt bump Co-authored-by: Marshall Krassenstein --- .github/workflows/dependency-audit.yml | 11 +++++++ .github/workflows/update-lockfile.yml | 40 ++++++++++++++++++++++++++ .gitignore | 3 ++ pyproject.toml | 18 +++++++++++- 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/update-lockfile.yml diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index b152db0b..afe085ef 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -30,6 +30,17 @@ jobs: - name: Install audit tools run: pip install pip-audit==2.9.0 uv==0.7.12 + - name: Compile requirements.txt from pyproject.toml + run: | + # Keep requirements.txt in sync with pyproject.toml so Dependabot can scan it. + # Note: [tool.uv.sources] git overrides are not resolved by pip compile — + # requests and cryptography fall back to their PyPI versions here, which is + # intentional for Dependabot's purposes. + uv pip compile pyproject.toml -o /tmp/requirements.compiled.txt + if ! diff -q requirements.txt /tmp/requirements.compiled.txt > /dev/null 2>&1; then + echo "::warning::requirements.txt is out of date with pyproject.toml. Run: uv pip compile pyproject.toml -o requirements.txt" + fi + - name: Audit pinned dependencies run: | if [ -f requirements.lock ]; then diff --git a/.github/workflows/update-lockfile.yml b/.github/workflows/update-lockfile.yml new file mode 100644 index 00000000..ef656d5e --- /dev/null +++ b/.github/workflows/update-lockfile.yml @@ -0,0 +1,40 @@ +name: Update Lockfile + +on: + push: + branches: [main] + paths: + - "requirements.txt" + +jobs: + update-lockfile: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + + - name: Install uv + run: pip install uv==0.7.12 + + - name: Regenerate requirements.lock + run: uv pip compile requirements.txt -o requirements.lock --generate-hashes + + - name: Commit updated lockfile + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if git diff --quiet requirements.lock; then + echo "requirements.lock is already up to date, nothing to commit" + else + git add requirements.lock + git commit -m "chore: regenerate requirements.lock after requirements.txt update" + git push + fi diff --git a/.gitignore b/.gitignore index 33ee1c8d..f9acd43e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ outstanding-todos.md # Uploaded files (clipboard paste images) uploads/ + +# uv lockfile — not portable across PyPI proxies, generate locally with `uv lock` +uv.lock diff --git a/pyproject.toml b/pyproject.toml index c60f459d..06e4dbbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,24 @@ description = "CoDA - Coding Agents on Databricks Apps" requires-python = ">=3.10" dependencies = [ "flask>=2.0", + "flask-socketio>=5.0", + "simple-websocket>=1.0", "claude-agent-sdk", "databricks-sdk>=0.20.0", - "mlflow[genai]>=3.4", + "mlflow-tracing>=3.4", "opentelemetry-exporter-otlp-proto-grpc", + "requests", + "cryptography", ] + +[tool.uv] +# Exclude packages uploaded to PyPI more recently than ~30 days ago. +# This gives the community time to catch supply-chain issues before they land here. +# Bump this date when you intentionally need a newer release. +exclude-newer = "7 days" + +[tool.uv.sources] +# Direct GitHub installs — workaround for Databricks internal PyPI proxy gaps. +# Remove these once the proxy has current versions. +requests = { git = "https://github.com/psf/requests", rev = "v2.33.0" } +cryptography = { git = "https://github.com/pyca/cryptography", rev = "46.0.6" } From 30997c06d88df40fa23ce605fb6b316f30c2cd81 Mon Sep 17 00:00:00 2001 From: Marshall Krassenstein Date: Mon, 30 Mar 2026 16:56:29 -0400 Subject: [PATCH 156/382] fix: use astral-sh/setup-uv action instead of pinned pip install uv==0.7.12 predates relative duration support in exclude-newer ("7 days"). Switching to the official action ensures we always get a current uv version. Co-authored-by: Marshall Krassenstein --- .github/workflows/dependency-audit.yml | 5 ++++- .github/workflows/update-lockfile.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index afe085ef..9f62ff3d 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -27,8 +27,11 @@ jobs: with: python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Install audit tools - run: pip install pip-audit==2.9.0 uv==0.7.12 + run: pip install pip-audit==2.9.0 - name: Compile requirements.txt from pyproject.toml run: | diff --git a/.github/workflows/update-lockfile.yml b/.github/workflows/update-lockfile.yml index ef656d5e..f96549e9 100644 --- a/.github/workflows/update-lockfile.yml +++ b/.github/workflows/update-lockfile.yml @@ -22,7 +22,7 @@ jobs: python-version: "3.11" - name: Install uv - run: pip install uv==0.7.12 + uses: astral-sh/setup-uv@v5 - name: Regenerate requirements.lock run: uv pip compile requirements.txt -o requirements.lock --generate-hashes From dc54f9ac452672211ab93e134b738ecf4ba38b77 Mon Sep 17 00:00:00 2001 From: Marshall Krassenstein Date: Wed, 1 Apr 2026 07:30:47 -0400 Subject: [PATCH 157/382] fix: switch cryptography from git source to PyPI 46.0.6 Co-authored-by: Marshall Krassenstein --- pyproject.toml | 1 - requirements.lock | 2 +- requirements.txt | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 06e4dbbe..2ec0ad2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,4 +25,3 @@ exclude-newer = "7 days" # Direct GitHub installs — workaround for Databricks internal PyPI proxy gaps. # Remove these once the proxy has current versions. requests = { git = "https://github.com/psf/requests", rev = "v2.33.0" } -cryptography = { git = "https://github.com/pyca/cryptography", rev = "46.0.6" } diff --git a/requirements.lock b/requirements.lock index d4088233..9d5bd67a 100644 --- a/requirements.lock +++ b/requirements.lock @@ -256,7 +256,7 @@ click==8.3.1 \ # flask # flask-socketio # uvicorn -cryptography @ git+https://github.com/pyca/cryptography@91d728897bdad30cd5c79a2b23e207f1f050d587 +cryptography==46.0.6 # via # -r requirements.txt # pyjwt diff --git a/requirements.txt b/requirements.txt index 8fdb2f74..f488270f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ databricks-sdk==0.102.0 mlflow-tracing==3.10.1 opentelemetry-exporter-otlp-proto-grpc==1.40.0 requests @ git+https://github.com/psf/requests@v2.33.0 -cryptography @ git+https://github.com/pyca/cryptography@46.0.6 +cryptography==46.0.6 From aa2b749f687460a8dc3a6ddcdf999f7c79c0fef3 Mon Sep 17 00:00:00 2001 From: Marshall Krassenstein Date: Wed, 1 Apr 2026 09:43:09 -0400 Subject: [PATCH 158/382] Update makefile to remove all of the secret scope stuff --- Makefile | 101 +++++++++---------------------------------------------- 1 file changed, 15 insertions(+), 86 deletions(-) diff --git a/Makefile b/Makefile index a35a9655..86b5e4a5 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,28 @@ # Makefile for deploying Coding Agents to Databricks Apps # # Usage: -# make deploy-e2e PROFILE=dogfood # fully automated deploy (auto-generates PAT) -# make deploy PROFILE=dogfood # full deploy (prompts for PAT interactively) -# make redeploy PROFILE=dogfood # skip secret setup, just sync + deploy +# make deploy PROFILE=dogfood # full deploy (create app, sync, deploy) +# make redeploy PROFILE=dogfood # skip app creation, just sync + deploy +# make create-pat PROFILE=dogfood # generate a 1-day PAT and copy to clipboard # make status PROFILE=dogfood # check app status # make open PROFILE=dogfood # open app in browser # make clean PROFILE=dogfood # remove app and secret scope -# Configuration (accepts lowercase: make deploy-e2e profile=dogfood) +# Configuration (accepts lowercase: make deploy profile=dogfood) ifdef profile PROFILE := $(profile) endif ifdef app_name APP_NAME := $(app_name) endif -ifdef pat -PAT := $(pat) -endif PROFILE ?= DEFAULT APP_NAME ?= coding-agents -SECRET_SCOPE ?= $(APP_NAME)-secrets -SECRET_KEY ?= databricks-token # Resolve user email and workspace path from the profile USER_EMAIL = $(shell databricks current-user me --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('userName',''))") WORKSPACE_PATH = /Workspace/Users/$(USER_EMAIL)/apps/$(APP_NAME) -.PHONY: help deploy-e2e deploy redeploy create-app create-pat setup-secret sync deploy-app status open clean clean-secret +.PHONY: help deploy redeploy create-app create-pat sync deploy-app status open clean # ── Help ───────────────────────────────────────────── @@ -36,12 +31,7 @@ help: ## Show this help # ── Workflows ──────────────────────────────────────── -deploy-e2e: create-app create-pat sync deploy-app ## Full automated deploy (auto-generates PAT) - @echo "" - @echo "Deployment complete! App URL:" - @databricks apps get $(APP_NAME) --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('url','(pending)'))" - -deploy: create-app setup-secret sync deploy-app ## Full deploy (prompts for PAT interactively) +deploy: create-app sync deploy-app ## Full deploy (create app, sync, deploy) @echo "" @echo "Deployment complete! App URL:" @databricks apps get $(APP_NAME) --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('url','(pending)'))" @@ -73,55 +63,13 @@ create-app: ## Create the Databricks App (idempotent) databricks apps create $(APP_NAME) --profile $(PROFILE); \ fi -create-pat: ## Generate a 90-day PAT and store it as the app secret - @echo "==> Ensuring secret scope '$(SECRET_SCOPE)' exists..." - @if databricks secrets list-scopes --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); scopes=[s['name'] for s in (d if isinstance(d,list) else d.get('scopes',[]))]; exit(0 if '$(SECRET_SCOPE)' in scopes else 1)" 2>/dev/null; then \ - echo " Secret scope '$(SECRET_SCOPE)' already exists."; \ - else \ - echo " Creating secret scope '$(SECRET_SCOPE)'..."; \ - databricks secrets create-scope $(SECRET_SCOPE) --profile $(PROFILE); \ - fi - @echo "==> Generating a 90-day PAT..." - @databricks tokens create --lifetime-seconds $$((90 * 24 * 60 * 60)) --comment "coding-agents (auto-generated)" --profile $(PROFILE) --output json \ - | python3 -c "import sys,json; print(json.load(sys.stdin)['token_value'])" \ - | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE) - @echo " PAT created and stored in $(SECRET_SCOPE)/$(SECRET_KEY)" - @echo "==> Linking secret to app resource 'DATABRICKS_TOKEN'..." - @curl -s -X PATCH \ - "$$(databricks auth env --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['env']['DATABRICKS_HOST'])")/api/2.0/apps/$(APP_NAME)" \ - -H "Authorization: Bearer $$(databricks auth token --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")" \ - -H "Content-Type: application/json" \ - -d '{"resources":[{"name":"DATABRICKS_TOKEN","description":"PAT for model serving access","secret":{"scope":"$(SECRET_SCOPE)","key":"$(SECRET_KEY)","permission":"READ"}}]}' \ - >/dev/null - @echo " App resource linked." - -setup-secret: ## Create secret scope and store PAT (interactive) - @echo "==> Setting up DATABRICKS_TOKEN secret..." - @# Create scope if it doesn't exist - @if databricks secrets list-scopes --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); scopes=[s['name'] for s in (d if isinstance(d,list) else d.get('scopes',[]))]; exit(0 if '$(SECRET_SCOPE)' in scopes else 1)" 2>/dev/null; then \ - echo " Secret scope '$(SECRET_SCOPE)' already exists."; \ - else \ - echo " Creating secret scope '$(SECRET_SCOPE)'..."; \ - databricks secrets create-scope $(SECRET_SCOPE) --profile $(PROFILE); \ - fi - @# Store the PAT - prompt if not provided - @if [ -z "$(PAT)" ]; then \ - echo " Enter your Databricks PAT (will not echo):"; \ - read -s pat_value && \ - echo "$$pat_value" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \ - else \ - echo "$(PAT)" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \ - fi - @echo " Secret stored in $(SECRET_SCOPE)/$(SECRET_KEY)" - @# Link secret to app resource - @echo " Linking secret to app resource 'DATABRICKS_TOKEN'..." - @curl -s -X PATCH \ - "$$(databricks auth env --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['env']['DATABRICKS_HOST'])")/api/2.0/apps/$(APP_NAME)" \ - -H "Authorization: Bearer $$(databricks auth token --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")" \ - -H "Content-Type: application/json" \ - -d '{"resources":[{"name":"DATABRICKS_TOKEN","description":"PAT for model serving access","secret":{"scope":"$(SECRET_SCOPE)","key":"$(SECRET_KEY)","permission":"READ"}}]}' \ - >/dev/null - @echo " App resource linked." +create-pat: ## Generate a 1-day PAT and copy it to your clipboard + @echo "==> Generating a 1-day PAT..." + @token=$$(databricks tokens create --lifetime-seconds $$((1 * 24 * 60 * 60)) --comment "coding-agents (1-day)" --profile $(PROFILE) --output json \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['token_value'])") && \ + echo "$$token" | pbcopy && \ + echo " PAT copied to clipboard! (expires in 24 hours)" + sync: ## Sync local files to Databricks workspace @echo "==> Syncing to $(WORKSPACE_PATH)..." @@ -143,28 +91,9 @@ open: ## Open the app in browser # ── Cleanup (destructive) ─────────────────────────── -clean: ## Remove app and secret scope (destructive) +clean: ## Remove the app (destructive) @echo "==> Removing app '$(APP_NAME)'..." @databricks apps delete $(APP_NAME) --profile $(PROFILE) 2>/dev/null && \ echo " App '$(APP_NAME)' deleted." || \ echo " App '$(APP_NAME)' not found or already deleted." - @echo "==> Removing secret scope '$(SECRET_SCOPE)'..." - @databricks secrets delete-scope $(SECRET_SCOPE) --profile $(PROFILE) 2>/dev/null && \ - echo " Secret scope '$(SECRET_SCOPE)' deleted." || \ - echo " Secret scope '$(SECRET_SCOPE)' not found or already deleted." - -clean-secret: ## Remove secret and optionally the scope (destructive) - @echo "==> Removing secret '$(SECRET_KEY)' from scope '$(SECRET_SCOPE)'..." - @databricks secrets delete-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE) 2>/dev/null && \ - echo " Secret '$(SECRET_KEY)' deleted." || \ - echo " Secret '$(SECRET_KEY)' not found or already deleted." - @printf " Remove the entire secret scope '$(SECRET_SCOPE)'? [y/N] " && \ - read answer && \ - if [ "$$answer" = "y" ] || [ "$$answer" = "Y" ]; then \ - echo " Removing secret scope '$(SECRET_SCOPE)'..."; \ - databricks secrets delete-scope $(SECRET_SCOPE) --profile $(PROFILE) && \ - echo " Secret scope '$(SECRET_SCOPE)' deleted." || \ - echo " Failed to delete secret scope."; \ - else \ - echo " Keeping secret scope '$(SECRET_SCOPE)'."; \ - fi + From 85444f7b9f96a8ef74b5f28b0e0bb7bb76895fa5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Apr 2026 05:35:44 +0000 Subject: [PATCH 159/382] chore(deps): bump astral-sh/setup-uv from 5 to 7 Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 5 to 7. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/v5...v7) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dependency-audit.yml | 2 +- .github/workflows/update-lockfile.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index 9f62ff3d..a4d7cd07 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -28,7 +28,7 @@ jobs: python-version: "3.11" - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 - name: Install audit tools run: pip install pip-audit==2.9.0 diff --git a/.github/workflows/update-lockfile.yml b/.github/workflows/update-lockfile.yml index f96549e9..98a7b207 100644 --- a/.github/workflows/update-lockfile.yml +++ b/.github/workflows/update-lockfile.yml @@ -22,7 +22,7 @@ jobs: python-version: "3.11" - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 - name: Regenerate requirements.lock run: uv pip compile requirements.txt -o requirements.lock --generate-hashes From a39147922db74e9efa90db2e812d38deffb9362f Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 5 Apr 2026 18:34:25 -0400 Subject: [PATCH 160/382] docs: sync README with current repo state (#96) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: sync README with current repo state - Remove defunct loading screen / snake game references - Update architecture diagram: 11 steps (5 sequential → 6 parallel) - Add missing files to project structure (app_state, cli_auth, content_filter_proxy, pat_rotator, setup_proxy, install scripts) - Document TEAM_MEMORY_MCP_URL env var and optional team-memory MCP - Add pyproject.toml, requirements.lock, Makefile, CI workflows - Replace loading.html with favicon.svg in static listing - Add uv to technologies line * docs: remove undocumented TEAM_MEMORY_MCP_URL from README --- README.md | 78 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 3ff86e19..922ef8b4 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ This isn't just a terminal in the cloud. Running coding agents on Databricks giv | 🎤 **Voice Input** | Dictate commands with your mic (Option+V) | | 📋 **Image Paste** | Paste or drag-and-drop images into the terminal — saved to `~/uploads/`, path inserted automatically | | ⌨️ **Customizable** | Fonts, font sizes, themes — all persisted across sessions | -| 🐍 **Loading Screen** | Play snake while setup steps run in parallel | | 🔄 **Workspace Sync** | Every `git commit` auto-syncs to `/Workspace/Users/{you}/projects/` | | ✏️ **Micro Editor** | Modern terminal editor, pre-installed | | ⚙️ **Databricks CLI** | Installed at boot, configured interactively on first session | @@ -204,6 +203,7 @@ This template repo opens that vision up for every Databricks user — no IDE set | **DeepWiki** | Ask questions about any GitHub repo — gets AI-powered answers from the codebase | | **Exa** | Web search and code context retrieval for up-to-date information | +
@@ -221,8 +221,8 @@ This template repo opens that vision up for every Databricks user — no IDE set │ on first load │ on startup ▼ ▼ ┌─────────────────────┐ ┌─────────────────────┐ -│ Loading Screen │ │ Background Setup │ -│ (snake game) │ │ (8 steps, 6 ║) │ +│ Setup Progress │ │ Background Setup │ +│ (inline UI) │ │ (11 steps, 5→6 ║) │ └─────────────────────┘ └─────────────────────┘ │ ▼ @@ -235,18 +235,18 @@ This template repo opens that vision up for every Databricks user — no IDE set ### Startup Flow 1. Gunicorn starts, calls `initialize_app()` via `post_worker_init` hook -2. App immediately serves the loading screen (snake game) -3. Background thread runs setup: git config and micro editor run sequentially, then 6 agent setups (Claude, Codex, OpenCode, Gemini, Databricks CLI, MLflow) run in parallel via `ThreadPoolExecutor` -4. `/api/setup-status` endpoint reports progress to the loading screen -5. Once complete, the loading screen transitions to the terminal UI +2. App serves the terminal UI with inline setup progress +3. Background thread runs setup: 5 sequential steps (git config, micro editor, GitHub CLI, Databricks CLI upgrade, content-filter proxy), then 6 agent setups (Claude, Codex, OpenCode, Gemini, Databricks CLI config, MLflow) run in parallel via `ThreadPoolExecutor` +4. `/api/setup-status` endpoint reports progress to the UI +5. Once complete, the terminal becomes interactive ### API Endpoints | Endpoint | Method | Description | |----------|--------|-------------| -| `/` | GET | Loading screen (during setup) or terminal UI | +| `/` | GET | Terminal UI with inline setup progress | | `/health` | GET | Health check with session count and setup status | -| `/api/setup-status` | GET | Setup progress for loading screen | +| `/api/setup-status` | GET | Setup progress for the UI | | `/api/version` | GET | App version | | `/api/session` | POST | Create new terminal session | | `/api/input` | POST | Send input to terminal | @@ -302,32 +302,46 @@ Production uses `workers=1` (PTY state is process-local), `threads=16` (concurre ``` coding-agents-in-databricks/ -├── app.py # Flask backend + PTY management + setup orchestration -├── app.yaml.template # Databricks Apps deployment config template -├── gunicorn.conf.py # Gunicorn production server config -├── requirements.txt # Python dependencies -├── setup_claude.py # Claude Code CLI + MCP configuration -├── setup_codex.py # Codex CLI configuration -├── setup_gemini.py # Gemini CLI configuration -├── setup_opencode.py # OpenCode configuration -├── setup_databricks.py # Databricks CLI configuration -├── setup_mlflow.py # MLflow tracing auto-configuration -├── sync_to_workspace.py # Post-commit hook: sync to Workspace -├── install_micro.sh # Micro editor installer -├── utils.py # Utility functions (ensure_https) +├── app.py # Flask backend + PTY management + setup orchestration +├── app_state.py # Shared app state (setup progress, session registry) +├── app.yaml.template # Databricks Apps deployment config template +├── cli_auth.py # Interactive PAT setup + CLI credential writer +├── content_filter_proxy.py # Proxy that sanitises empty-content blocks for OpenCode +├── gunicorn.conf.py # Gunicorn production server config +├── pat_rotator.py # Background PAT auto-rotation (10-min cycle) +├── pyproject.toml # Package metadata + uv config (supply-chain guardrails) +├── requirements.txt # Compiled from pyproject.toml (Dependabot compatibility) +├── requirements.lock # Hash-pinned lockfile (auto-regenerated by CI) +├── Makefile # Deploy, redeploy, status, and cleanup targets +├── setup_claude.py # Claude Code CLI + MCP configuration +├── setup_codex.py # Codex CLI configuration +├── setup_gemini.py # Gemini CLI configuration +├── setup_opencode.py # OpenCode configuration +├── setup_databricks.py # Databricks CLI configuration +├── setup_mlflow.py # MLflow tracing auto-configuration +├── setup_proxy.py # Content-filter proxy startup +├── sync_to_workspace.py # Post-commit hook: sync to Workspace +├── install_micro.sh # Micro editor installer +├── install_gh.sh # GitHub CLI installer (OS/arch-aware) +├── install_databricks_cli.sh # Databricks CLI upgrade script +├── utils.py # Utility functions (ensure_https) ├── static/ -│ ├── index.html # Terminal UI (xterm.js + split panes + WebSocket) -│ ├── loading.html # Loading screen with snake game -│ ├── poll-worker.js # Web Worker for HTTP polling fallback +│ ├── index.html # Terminal UI (xterm.js + split panes + WebSocket) +│ ├── favicon.svg # App favicon +│ ├── poll-worker.js # Web Worker for HTTP polling fallback │ └── lib/ -│ ├── xterm.js # xterm.js terminal emulator -│ └── socket.io.min.js # Vendored Socket.IO client +│ ├── xterm.js # xterm.js terminal emulator +│ └── socket.io.min.js # Vendored Socket.IO client ├── .claude/ -│ └── skills/ # 39 pre-installed skills +│ └── skills/ # 39 pre-installed skills +├── .github/ +│ └── workflows/ +│ ├── dependency-audit.yml # Weekly CVE audit + lockfile drift check +│ └── update-lockfile.yml # Auto-regenerate requirements.lock on push └── docs/ - ├── deployment.md # Full Databricks Apps deployment guide - ├── prd/ # Product requirement documents - └── plans/ # Design documentation + ├── deployment.md # Full Databricks Apps deployment guide + ├── prd/ # Product requirement documents + └── plans/ # Design documentation ```
@@ -336,4 +350,4 @@ coding-agents-in-databricks/ ## Technologies -Flask · Flask-SocketIO · Socket.IO · Gunicorn · xterm.js · Python PTY · Databricks SDK · Databricks AI Gateway · MLflow \ No newline at end of file +Flask · Flask-SocketIO · Socket.IO · Gunicorn · xterm.js · Python PTY · uv · Databricks SDK · Databricks AI Gateway · MLflow From bdb838dab1b88f7092495ca78a80e096364eba24 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 5 Apr 2026 19:08:33 -0400 Subject: [PATCH 161/382] fix: add workflow_dispatch to audit + revoke bootstrap PAT (#99) * fix: add workflow_dispatch to audit + revoke bootstrap PAT (#97, #98) - Add workflow_dispatch trigger to dependency-audit.yml so it can be run on-demand from the GitHub Actions tab - After the first successful PAT rotation, list all pre-existing tokens and revoke them (including the bootstrap PAT the user pasted). This ensures no stale tokens sit around after the app has its own controlled short-lived token. * fix: revoke only the bootstrap PAT, not all existing tokens Instead of revoking every token except the freshly minted one (which nuked the user's other PATs for notebooks, CI, etc.), identify the bootstrap PAT as the most-recently-created token without a "coda-auto-rotated" comment and revoke only that one. --- .github/workflows/dependency-audit.yml | 1 + app.py | 3 +- pat_rotator.py | 58 ++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index a4d7cd07..097ffaf0 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -1,6 +1,7 @@ name: Dependency Audit on: + workflow_dispatch: pull_request: paths: - "requirements.txt" diff --git a/app.py b/app.py index 671fd1d2..56094b8c 100644 --- a/app.py +++ b/app.py @@ -921,13 +921,14 @@ def configure_pat(): # Immediately mint a controlled short-lived token from the user-pasted PAT. # This gives us a token ID we own — all future rotations can revoke the old one. - # The user-pasted PAT becomes unused after this (expires per its own lifetime). os.environ["DATABRICKS_TOKEN"] = token pat_rotator._current_token = token pat_rotator._current_token_id = None rotated = pat_rotator._rotate_once() if rotated: token = pat_rotator.token # use the newly minted token from here on + # Revoke only the bootstrap PAT — leave other user PATs intact (#98) + pat_rotator.revoke_bootstrap_token() else: # Rotation failed — fall back to user-pasted token (still valid) pat_rotator._write_databrickscfg(token) diff --git a/pat_rotator.py b/pat_rotator.py index d3b6ba80..28e0319d 100644 --- a/pat_rotator.py +++ b/pat_rotator.py @@ -161,6 +161,64 @@ def _rotate_once(self): return True + def revoke_bootstrap_token(self): + """Revoke only the bootstrap PAT after the first rotation. + + Called once after the bootstrap PAT is replaced by a controlled + short-lived token. Lists all tokens, identifies the bootstrap + as the most-recently-created token without a "coda-auto-rotated" + comment, and revokes only that one. Other user PATs (notebooks, + CI, etc.) are left untouched. + """ + current_id = self._current_token_id + token = self._current_token + if not token or not current_id: + return + + try: + resp = requests.get( + f"{self._host}/api/2.0/token/list", + headers={"Authorization": f"Bearer {token}"}, + timeout=30 + ) + if resp.status_code != 200: + logger.warning(f"Bootstrap cleanup: failed to list tokens ({resp.status_code})") + return + except requests.RequestException as e: + logger.warning(f"Bootstrap cleanup: list request failed: {e}") + return + + token_infos = resp.json().get("token_infos", []) + + # Find the bootstrap PAT: newest non-coda token that isn't the current one + candidates = [ + info for info in token_infos + if info.get("token_id") != current_id + and info.get("comment", "") != "coda-auto-rotated" + ] + if not candidates: + logger.info("Bootstrap cleanup: no bootstrap token candidate found") + return + + # The bootstrap PAT is the most recently created candidate + bootstrap = max(candidates, key=lambda t: t.get("creation_time", 0)) + tid = bootstrap.get("token_id") + comment = bootstrap.get("comment", "(no comment)") + + try: + del_resp = requests.post( + f"{self._host}/api/2.0/token/delete", + headers={"Authorization": f"Bearer {token}"}, + json={"token_id": tid}, + timeout=30 + ) + if del_resp.status_code == 200: + logger.info(f"Bootstrap cleanup: revoked bootstrap PAT {tid} ({comment})") + else: + logger.warning(f"Bootstrap cleanup: failed to revoke {tid} ({del_resp.status_code})") + except requests.RequestException as e: + logger.warning(f"Bootstrap cleanup: revoke request failed: {e}") + def _persist_token(self, token): """Write rotated token to all persistence layers.""" os.environ["DATABRICKS_TOKEN"] = token From adbc45146a9f7abf7c23c34a6950df6a142ef65c Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Sun, 5 Apr 2026 19:13:11 -0400 Subject: [PATCH 162/382] chore: bump version to 0.16.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2ec0ad2d..eedde9fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coda" -version = "0.16.5" +version = "0.16.6" description = "CoDA - Coding Agents on Databricks Apps" requires-python = ">=3.10" dependencies = [ From 814274ace8421b890f6c4e8785c94e446d2cfd8e Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Tue, 7 Apr 2026 19:33:10 +1000 Subject: [PATCH 163/382] fix: add model tier mappings and disable experimental betas in Claude settings The Databricks AI Gateway proxy rejects experimental API fields like cache_control.scope that newer Claude Code versions send. Also adds ANTHROPIC_DEFAULT_*_MODEL env vars so Claude Code correctly maps its internal model tiers (opus/sonnet/haiku) to Databricks endpoint names. Co-authored-by: Isaac --- app.py | 6 +++++- setup_claude.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 671fd1d2..94af1f3a 100644 --- a/app.py +++ b/app.py @@ -288,10 +288,14 @@ def _configure_all_cli_auth(token): settings = { "env": { - "ANTHROPIC_MODEL": os.environ.get("ANTHROPIC_MODEL", "databricks-claude-sonnet-4-6"), + "ANTHROPIC_MODEL": os.environ.get("ANTHROPIC_MODEL", "databricks-claude-opus-4-6"), "ANTHROPIC_BASE_URL": anthropic_base_url, "ANTHROPIC_AUTH_TOKEN": token, + "ANTHROPIC_DEFAULT_OPUS_MODEL": "databricks-claude-opus-4-6", + "ANTHROPIC_DEFAULT_SONNET_MODEL": "databricks-claude-sonnet-4-6", + "ANTHROPIC_DEFAULT_HAIKU_MODEL": "databricks-claude-haiku-4-5", "ANTHROPIC_CUSTOM_HEADERS": "x-databricks-use-coding-agent-mode: true", + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "1", } } diff --git a/setup_claude.py b/setup_claude.py index d13dfcb9..cf7e3c8a 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -32,10 +32,14 @@ settings = { "env": { - "ANTHROPIC_MODEL": os.environ.get("ANTHROPIC_MODEL", "databricks-claude-sonnet-4-6"), + "ANTHROPIC_MODEL": os.environ.get("ANTHROPIC_MODEL", "databricks-claude-opus-4-6"), "ANTHROPIC_BASE_URL": anthropic_base_url, "ANTHROPIC_AUTH_TOKEN": token, - "ANTHROPIC_CUSTOM_HEADERS": "x-databricks-use-coding-agent-mode: true" + "ANTHROPIC_DEFAULT_OPUS_MODEL": "databricks-claude-opus-4-6", + "ANTHROPIC_DEFAULT_SONNET_MODEL": "databricks-claude-sonnet-4-6", + "ANTHROPIC_DEFAULT_HAIKU_MODEL": "databricks-claude-haiku-4-5", + "ANTHROPIC_CUSTOM_HEADERS": "x-databricks-use-coding-agent-mode: true", + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "1", } } From bc81c16f647e768d310527b30ed7cc00e3d9d536 Mon Sep 17 00:00:00 2001 From: Marshall Krassenstein Date: Tue, 7 Apr 2026 09:06:52 -0400 Subject: [PATCH 164/382] chore: add PR template with dogfood testing checklist Co-authored-by: Marshall Krassenstein --- .github/pull_request_template.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..efa5e8ba --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,10 @@ +## What does this PR do? + + + +## Testing + +Please test your changes on dogfood before merging. + +- [ ] Deployed and tested on dogfood +- [ ] Added a screenshot to this PR From 403fca8ecf879d983393efa093b147b3604f3b9f Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Wed, 8 Apr 2026 14:20:02 -0400 Subject: [PATCH 165/382] fix: inject fresh proxy token + strip DATABRICKS_HOST from shell env (#107) * fix: inject fresh token in proxy to survive PAT rotation Cherry-picked from PR #105 (dgokeeffe) for testing. * fix: strip DATABRICKS_HOST from shell env to unblock CLI auth The Databricks SDK skips ~/.databrickscfg when DATABRICKS_HOST is set in env (even without credentials). After DATABRICKS_TOKEN and SP credentials are stripped, the CLI sees host + workspace_id but no token and fails. Stripping DATABRICKS_HOST forces the SDK to fall through to ~/.databrickscfg which has both host and token (kept fresh by PAT rotator). Mirrors the pattern in sync_to_workspace.py. Ref #105 --- app.py | 7 ++++-- content_filter_proxy.py | 49 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index e18808cb..2fe0c8a2 100644 --- a/app.py +++ b/app.py @@ -966,9 +966,12 @@ def create_session(): # Remove Claude Code env vars so the browser terminal isn't seen as nested shell_env.pop("CLAUDECODE", None) shell_env.pop("CLAUDE_CODE_SESSION", None) - # Remove DATABRICKS_TOKEN so CLI/SDK reads from ~/.databrickscfg (always - # current after rotation) instead of inheriting a stale env var snapshot + # Remove DATABRICKS_TOKEN and DATABRICKS_HOST so CLI/SDK reads from + # ~/.databrickscfg (always current after rotation) instead of inheriting + # a stale env var snapshot. The SDK skips config file loading when + # DATABRICKS_HOST is set in env (even without credentials). shell_env.pop("DATABRICKS_TOKEN", None) + shell_env.pop("DATABRICKS_HOST", None) # Ensure HOME is set correctly if not shell_env.get("HOME") or shell_env["HOME"] == "/": shell_env["HOME"] = "/app/python/source_code" diff --git a/content_filter_proxy.py b/content_filter_proxy.py index d8303201..c20d8d2a 100644 --- a/content_filter_proxy.py +++ b/content_filter_proxy.py @@ -16,10 +16,12 @@ See: https://github.com/sst/opencode/issues/5028 https://github.com/BerriAI/litellm/pull/20384 """ +import configparser import json import logging import os import sys +import time from http.server import HTTPServer, BaseHTTPRequestHandler from socketserver import ThreadingMixIn @@ -29,6 +31,45 @@ LISTEN_HOST = os.environ.get("PROXY_HOST", "127.0.0.1") LISTEN_PORT = int(os.environ.get("PROXY_PORT", "4000")) +# --------------------------------------------------------------------------- +# Fresh token injection — survives PAT rotation +# --------------------------------------------------------------------------- +# The PAT rotator writes the latest token to ~/.databrickscfg every rotation. +# OpenCode (and this proxy) are separate processes with frozen env snapshots, +# so we read the file on-demand instead of trusting os.environ. + +_TOKEN_CACHE: dict = {"token": None, "read_at": 0.0} +_TOKEN_CACHE_TTL = 30 # seconds — short enough to pick up rotations quickly + +_HOME = os.environ.get("HOME", "/app/python/source_code") +if not _HOME or _HOME == "/": + _HOME = "/app/python/source_code" +_DATABRICKSCFG_PATH = os.path.join(_HOME, ".databrickscfg") + + +def _get_fresh_token() -> str | None: + """Read current token from ~/.databrickscfg (updated by PAT rotator). + + Returns cached value if read within the last _TOKEN_CACHE_TTL seconds. + """ + now = time.time() + if _TOKEN_CACHE["token"] and (now - _TOKEN_CACHE["read_at"]) < _TOKEN_CACHE_TTL: + return _TOKEN_CACHE["token"] + + try: + config = configparser.ConfigParser() + config.read(_DATABRICKSCFG_PATH) + token = config.get("DEFAULT", "token", fallback=None) + if token: + _TOKEN_CACHE["token"] = token + _TOKEN_CACHE["read_at"] = now + return token + except Exception as e: + log.warning(f"Could not read fresh token from {_DATABRICKSCFG_PATH}: {e}") + + return _TOKEN_CACHE.get("token") # stale is better than nothing + + # Diagnostic logging — writes to stderr which goes to ~/.content-filter-proxy.log log = logging.getLogger("content-filter-proxy") log.setLevel(logging.INFO) @@ -502,13 +543,19 @@ def do_POST(self): # Build upstream URL upstream_url = UPSTREAM_BASE + self.path - # Forward headers + # Forward headers (inject fresh token to survive PAT rotation) headers = {} for key in self.headers: if key.lower() not in ("host", "content-length", "transfer-encoding"): headers[key] = self.headers[key] headers["Content-Length"] = str(len(body)) + # Override auth with fresh token from disk — OpenCode's cached token + # goes stale after PAT rotation since it's a long-lived TUI process + fresh_token = _get_fresh_token() + if fresh_token: + headers["Authorization"] = f"Bearer {fresh_token}" + # Detect streaming is_stream = False try: From c7d50ae060b8f8340426b3e880f7f92e804cbe60 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Wed, 8 Apr 2026 15:22:56 -0400 Subject: [PATCH 166/382] Auto-discover AI Gateway host from DATABRICKS_WORKSPACE_ID (#104) * feat: auto-discover AI Gateway host from DATABRICKS_WORKSPACE_ID Add get_gateway_host() helper to utils.py that resolves the gateway URL with a 3-tier priority: explicit DATABRICKS_GATEWAY_HOST env var > auto-constructed from DATABRICKS_WORKSPACE_ID > empty (fallback to DATABRICKS_HOST/serving-endpoints). Remove DATABRICKS_GATEWAY_HOST from app.yaml since Databricks Apps auto-inject DATABRICKS_WORKSPACE_ID into every container. Closes #103 * test: add gateway auto-discovery and endpoint construction tests 15 tests covering get_gateway_host() priority logic (explicit override > workspace ID > empty fallback) and endpoint URL construction for all 4 service paths (anthropic, openai/v1, gemini, mlflow/v1). Ref #103 --- README.md | 2 +- app.py | 4 +- app.yaml | 2 - docs/deployment.md | 2 +- setup_claude.py | 5 +- setup_codex.py | 7 +- setup_gemini.py | 7 +- setup_opencode.py | 7 +- setup_proxy.py | 4 +- tests/test_gateway_discovery.py | 207 ++++++++++++++++++++++++++++++++ utils.py | 20 +++ 11 files changed, 244 insertions(+), 23 deletions(-) create mode 100644 tests/test_gateway_discovery.py diff --git a/README.md b/README.md index 922ef8b4..129174cf 100644 --- a/README.md +++ b/README.md @@ -285,7 +285,7 @@ This template repo opens that vision up for every Databricks user — no IDE set | `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | | `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-2`) | | `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | -| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL (recommended) | +| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL override. Auto-discovered from `DATABRICKS_WORKSPACE_ID` if unset | ### Security Model diff --git a/app.py b/app.py index 2fe0c8a2..514c4bca 100644 --- a/app.py +++ b/app.py @@ -21,7 +21,7 @@ import requests import app_state -from utils import ensure_https +from utils import ensure_https, get_gateway_host from pat_rotator import PATRotator # Sanitize DATABRICKS_TOKEN early — the platform sometimes injects trailing @@ -278,7 +278,7 @@ def _configure_all_cli_auth(token): claude_dir = os.path.join(home, ".claude") os.makedirs(claude_dir, exist_ok=True) - gateway_host = ensure_https(os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/")) + gateway_host = get_gateway_host() databricks_host = ensure_https(os.environ.get("DATABRICKS_HOST", "").rstrip("/")) if gateway_host: diff --git a/app.yaml b/app.yaml index c221b425..9bf75940 100644 --- a/app.yaml +++ b/app.yaml @@ -10,7 +10,5 @@ env: value: databricks-gemini-3-1-pro - name: CODEX_MODEL value: databricks-gpt-5-2 - - name: DATABRICKS_GATEWAY_HOST - valueFrom: DATABRICKS_GATEWAY_HOST - name: CLAUDE_CODE_DISABLE_AUTO_MEMORY value: 0 diff --git a/docs/deployment.md b/docs/deployment.md index 5a98a7a0..c7196b59 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -70,7 +70,7 @@ databricks apps deploy \ | `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | | `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-2`) | | `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | -| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL (recommended). Falls back to direct model serving if unset | +| `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL override. Auto-discovered from `DATABRICKS_WORKSPACE_ID` if unset. Falls back to direct model serving if neither is available | ## Security Model diff --git a/setup_claude.py b/setup_claude.py index cf7e3c8a..725ad4d7 100644 --- a/setup_claude.py +++ b/setup_claude.py @@ -4,7 +4,7 @@ import subprocess from pathlib import Path -from utils import ensure_https +from utils import ensure_https, get_gateway_host # Set HOME if not properly set if not os.environ.get("HOME") or os.environ["HOME"] == "/": @@ -19,8 +19,7 @@ # 1. Write settings.json for Databricks model serving (requires DATABRICKS_TOKEN) token = os.environ.get("DATABRICKS_TOKEN", "").strip() if token: - # Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST - gateway_host = ensure_https(os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/")) + gateway_host = get_gateway_host() databricks_host = ensure_https(os.environ.get("DATABRICKS_HOST", "").rstrip("/")) if gateway_host: diff --git a/setup_codex.py b/setup_codex.py index 0c1b7545..6f5238ec 100644 --- a/setup_codex.py +++ b/setup_codex.py @@ -12,7 +12,7 @@ import subprocess from pathlib import Path -from utils import adapt_instructions_file, ensure_https, get_npm_version +from utils import adapt_instructions_file, ensure_https, get_gateway_host, get_npm_version # Set HOME if not properly set if not os.environ.get("HOME") or os.environ["HOME"] == "/": @@ -56,11 +56,10 @@ # Strip trailing slash and ensure https:// prefix host = ensure_https(host.rstrip("/")) -# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST -gateway_host = ensure_https(os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/")) +gateway_host = get_gateway_host() gateway_token = os.environ.get("DATABRICKS_TOKEN", "") if gateway_host else "" if gateway_host and not gateway_token: - print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") + print("Warning: AI Gateway resolved but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") gateway_host = "" if gateway_host: diff --git a/setup_gemini.py b/setup_gemini.py index 56b23761..b6acb976 100644 --- a/setup_gemini.py +++ b/setup_gemini.py @@ -16,7 +16,7 @@ import subprocess from pathlib import Path -from utils import adapt_instructions_file, ensure_https, get_npm_version +from utils import adapt_instructions_file, ensure_https, get_gateway_host, get_npm_version # Set HOME if not properly set if not os.environ.get("HOME") or os.environ["HOME"] == "/": @@ -59,11 +59,10 @@ # Strip trailing slash and ensure https:// prefix host = ensure_https(host.rstrip("/")) -# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to DATABRICKS_HOST -gateway_host = ensure_https(os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/")) +gateway_host = get_gateway_host() gateway_token = os.environ.get("DATABRICKS_TOKEN", "") if gateway_host else "" if gateway_host and not gateway_token: - print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") + print("Warning: AI Gateway resolved but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") gateway_host = "" if gateway_host: diff --git a/setup_opencode.py b/setup_opencode.py index ce348263..0d791af1 100644 --- a/setup_opencode.py +++ b/setup_opencode.py @@ -11,7 +11,7 @@ import subprocess from pathlib import Path -from utils import ensure_https, get_npm_version +from utils import ensure_https, get_gateway_host, get_npm_version # content-filter proxy local proxy — sanitizes empty content blocks before reaching Databricks # (see https://github.com/sst/opencode/issues/5028) @@ -74,11 +74,10 @@ # Strip trailing slash and ensure https:// prefix host = ensure_https(host.rstrip("/")) -# Use DATABRICKS_GATEWAY_HOST if available (new AI Gateway), otherwise fall back to current gateway (DATABRICKS_HOST) -gateway_host = ensure_https(os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/")) +gateway_host = get_gateway_host() gateway_token = os.environ.get("DATABRICKS_TOKEN", "") if gateway_host else "" if gateway_host and not gateway_token: - print("Warning: DATABRICKS_GATEWAY_HOST set but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") + print("Warning: AI Gateway resolved but DATABRICKS_TOKEN missing, falling back to DATABRICKS_HOST") gateway_host = "" if gateway_host: diff --git a/setup_proxy.py b/setup_proxy.py index ed03f8b1..92edd3c1 100644 --- a/setup_proxy.py +++ b/setup_proxy.py @@ -18,7 +18,7 @@ from urllib.request import urlopen, Request from urllib.error import URLError -from utils import ensure_https +from utils import ensure_https, get_gateway_host PROXY_PORT = 4000 PROXY_HOST = "127.0.0.1" @@ -63,7 +63,7 @@ pid_path.unlink(missing_ok=True) # Databricks configuration -gateway_host = ensure_https(os.environ.get("DATABRICKS_GATEWAY_HOST", "").rstrip("/")) +gateway_host = get_gateway_host() host = ensure_https(os.environ.get("DATABRICKS_HOST", "").rstrip("/")) token = os.environ.get("DATABRICKS_TOKEN", "") diff --git a/tests/test_gateway_discovery.py b/tests/test_gateway_discovery.py new file mode 100644 index 00000000..d27a9511 --- /dev/null +++ b/tests/test_gateway_discovery.py @@ -0,0 +1,207 @@ +"""Tests for AI Gateway auto-discovery — utils.get_gateway_host() and endpoint construction.""" + +import os +import subprocess +import sys +from pathlib import Path +from unittest import mock + +import pytest + +# --------------------------------------------------------------------------- +# Unit tests for get_gateway_host() +# --------------------------------------------------------------------------- + + +class TestGetGatewayHost: + """Test the 3-tier priority logic in get_gateway_host().""" + + def _get_fn(self): + from utils import get_gateway_host + return get_gateway_host + + @mock.patch.dict(os.environ, { + "DATABRICKS_GATEWAY_HOST": "https://custom.gateway.com", + "DATABRICKS_WORKSPACE_ID": "12345", + }) + def test_explicit_override_wins(self): + """Tier 1: explicit DATABRICKS_GATEWAY_HOST takes priority over workspace ID.""" + assert self._get_fn()() == "https://custom.gateway.com" + + @mock.patch.dict(os.environ, { + "DATABRICKS_GATEWAY_HOST": "custom.gateway.com", + "DATABRICKS_WORKSPACE_ID": "12345", + }) + def test_explicit_override_gets_https(self): + """Tier 1: explicit value without https:// gets it added.""" + assert self._get_fn()() == "https://custom.gateway.com" + + @mock.patch.dict(os.environ, { + "DATABRICKS_GATEWAY_HOST": "https://custom.gateway.com/", + "DATABRICKS_WORKSPACE_ID": "12345", + }) + def test_explicit_override_trailing_slash_stripped(self): + """Tier 1: trailing slash is stripped from explicit value.""" + assert self._get_fn()() == "https://custom.gateway.com" + + @mock.patch.dict(os.environ, {"DATABRICKS_WORKSPACE_ID": "6280049833385130"}, clear=False) + def test_auto_construct_from_workspace_id(self): + """Tier 2: construct gateway URL from DATABRICKS_WORKSPACE_ID.""" + env = os.environ.copy() + env.pop("DATABRICKS_GATEWAY_HOST", None) + with mock.patch.dict(os.environ, env, clear=True): + result = self._get_fn()() + assert result == "https://6280049833385130.ai-gateway.cloud.databricks.com" + + @mock.patch.dict(os.environ, {}, clear=True) + def test_empty_when_nothing_set(self): + """Tier 3: returns empty string when neither env var is set.""" + assert self._get_fn()() == "" + + @mock.patch.dict(os.environ, {"DATABRICKS_GATEWAY_HOST": "", "DATABRICKS_WORKSPACE_ID": ""}) + def test_empty_when_both_blank(self): + """Tier 3: returns empty when both vars are set but blank.""" + assert self._get_fn()() == "" + + @mock.patch.dict(os.environ, {"DATABRICKS_GATEWAY_HOST": " ", "DATABRICKS_WORKSPACE_ID": "12345"}) + def test_whitespace_only_gateway_falls_through(self): + """Whitespace-only DATABRICKS_GATEWAY_HOST falls through to workspace ID.""" + assert self._get_fn()() == "https://12345.ai-gateway.cloud.databricks.com" + + @mock.patch.dict(os.environ, {"DATABRICKS_GATEWAY_HOST": "", "DATABRICKS_WORKSPACE_ID": " 99999 "}) + def test_workspace_id_whitespace_stripped(self): + """Leading/trailing whitespace in workspace ID is stripped.""" + assert self._get_fn()() == "https://99999.ai-gateway.cloud.databricks.com" + + +# --------------------------------------------------------------------------- +# Integration tests — verify endpoint URLs constructed by setup scripts +# --------------------------------------------------------------------------- + +SETUP_DIR = Path(__file__).parent.parent + + +class TestEndpointConstruction: + """Verify setup scripts construct correct endpoint URLs with gateway auto-discovery.""" + + def _run_setup(self, script_name, tmp_path, env_overrides=None): + """Run a setup script as subprocess and capture output.""" + env = { + "HOME": str(tmp_path), + "DATABRICKS_HOST": "https://test.cloud.databricks.com", + "DATABRICKS_TOKEN": "dapi_test_token", + "DATABRICKS_WORKSPACE_ID": "6280049833385130", + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(SETUP_DIR), + } + # Ensure DATABRICKS_GATEWAY_HOST is NOT set (test auto-discovery) + env.pop("DATABRICKS_GATEWAY_HOST", None) + if env_overrides: + env.update(env_overrides) + + # Create required dirs + (tmp_path / ".claude").mkdir(exist_ok=True) + + result = subprocess.run( + [sys.executable, str(SETUP_DIR / script_name)], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + return result + + def test_setup_claude_uses_gateway(self, tmp_path): + """setup_claude.py should use auto-discovered gateway for anthropic URL.""" + result = self._run_setup("setup_claude.py", tmp_path) + assert result.returncode == 0, f"stderr: {result.stderr}" + assert "AI Gateway" in result.stdout or "6280049833385130" in result.stdout + + # Verify settings.json has gateway-based URL + import json + settings_path = tmp_path / ".claude" / "settings.json" + if settings_path.exists(): + settings = json.loads(settings_path.read_text()) + base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") + assert "6280049833385130.ai-gateway.cloud.databricks.com" in base_url + assert base_url.endswith("/anthropic") + + def test_setup_claude_explicit_override(self, tmp_path): + """setup_claude.py should prefer explicit DATABRICKS_GATEWAY_HOST.""" + result = self._run_setup("setup_claude.py", tmp_path, { + "DATABRICKS_GATEWAY_HOST": "https://custom.gateway.example.com", + }) + assert result.returncode == 0, f"stderr: {result.stderr}" + + import json + settings_path = tmp_path / ".claude" / "settings.json" + if settings_path.exists(): + settings = json.loads(settings_path.read_text()) + base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") + assert "custom.gateway.example.com" in base_url + + def test_setup_claude_fallback_no_gateway(self, tmp_path): + """setup_claude.py falls back to DATABRICKS_HOST when no gateway available.""" + result = self._run_setup("setup_claude.py", tmp_path, { + "DATABRICKS_WORKSPACE_ID": "", # No workspace ID + }) + assert result.returncode == 0, f"stderr: {result.stderr}" + + import json + settings_path = tmp_path / ".claude" / "settings.json" + if settings_path.exists(): + settings = json.loads(settings_path.read_text()) + base_url = settings.get("env", {}).get("ANTHROPIC_BASE_URL", "") + assert "test.cloud.databricks.com/serving-endpoints/anthropic" in base_url + + def test_codex_gateway_url_construction(self): + """Codex endpoint should use gateway /openai/v1 path.""" + from utils import get_gateway_host + with mock.patch.dict(os.environ, { + "DATABRICKS_WORKSPACE_ID": "6280049833385130", + }, clear=False): + env = os.environ.copy() + env.pop("DATABRICKS_GATEWAY_HOST", None) + with mock.patch.dict(os.environ, env, clear=True): + gw = get_gateway_host() + codex_url = f"{gw}/openai/v1" + assert codex_url == "https://6280049833385130.ai-gateway.cloud.databricks.com/openai/v1" + + def test_gemini_gateway_url_construction(self): + """Gemini endpoint should use gateway /gemini path.""" + from utils import get_gateway_host + with mock.patch.dict(os.environ, { + "DATABRICKS_WORKSPACE_ID": "6280049833385130", + }, clear=False): + env = os.environ.copy() + env.pop("DATABRICKS_GATEWAY_HOST", None) + with mock.patch.dict(os.environ, env, clear=True): + gw = get_gateway_host() + gemini_url = f"{gw}/gemini" + assert gemini_url == "https://6280049833385130.ai-gateway.cloud.databricks.com/gemini" + + def test_anthropic_gateway_url_construction(self): + """Anthropic endpoint should use gateway /anthropic path.""" + from utils import get_gateway_host + with mock.patch.dict(os.environ, { + "DATABRICKS_WORKSPACE_ID": "6280049833385130", + }, clear=False): + env = os.environ.copy() + env.pop("DATABRICKS_GATEWAY_HOST", None) + with mock.patch.dict(os.environ, env, clear=True): + gw = get_gateway_host() + anthropic_url = f"{gw}/anthropic" + assert anthropic_url == "https://6280049833385130.ai-gateway.cloud.databricks.com/anthropic" + + def test_proxy_gateway_url_construction(self): + """Proxy endpoint should use gateway /mlflow/v1 path.""" + from utils import get_gateway_host + with mock.patch.dict(os.environ, { + "DATABRICKS_WORKSPACE_ID": "6280049833385130", + }, clear=False): + env = os.environ.copy() + env.pop("DATABRICKS_GATEWAY_HOST", None) + with mock.patch.dict(os.environ, env, clear=True): + gw = get_gateway_host() + proxy_url = f"{gw}/mlflow/v1" + assert proxy_url == "https://6280049833385130.ai-gateway.cloud.databricks.com/mlflow/v1" diff --git a/utils.py b/utils.py index 7c3690b9..3e6c103e 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,6 @@ """Shared utilities for Databricks App setup scripts.""" +import os import re import subprocess from pathlib import Path @@ -60,6 +61,25 @@ def adapt_instructions_file( return True +def get_gateway_host() -> str: + """Resolve the AI Gateway host URL. + + Priority: + 1. Explicit DATABRICKS_GATEWAY_HOST env var (override) + 2. Auto-constructed from DATABRICKS_WORKSPACE_ID + 3. Empty string (caller falls back to DATABRICKS_HOST/serving-endpoints) + """ + explicit = os.environ.get("DATABRICKS_GATEWAY_HOST", "").strip().rstrip("/") + if explicit: + return ensure_https(explicit) + + workspace_id = os.environ.get("DATABRICKS_WORKSPACE_ID", "").strip() + if workspace_id: + return f"https://{workspace_id}.ai-gateway.cloud.databricks.com" + + return "" + + def ensure_https(url: str) -> str: """Ensure a URL has the https:// prefix. From 05892aadd501d2ad21a5a3d5fe23ffc66db6cb71 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Wed, 8 Apr 2026 17:47:01 -0400 Subject: [PATCH 167/382] chore: update default codex model to databricks-gpt-5-3-codex (#108) Bump version to 0.16.7. Update all references across app.yaml, setup_codex.py, setup_opencode.py, model-serving skill, README, and deployment docs. Closes #102 --- .claude/skills/databricks-model-serving/SKILL.md | 3 ++- README.md | 2 +- app.yaml | 2 +- docs/deployment.md | 2 +- pyproject.toml | 2 +- setup_codex.py | 2 +- setup_opencode.py | 6 +++--- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.claude/skills/databricks-model-serving/SKILL.md b/.claude/skills/databricks-model-serving/SKILL.md index 9c248aa9..de566f42 100644 --- a/.claude/skills/databricks-model-serving/SKILL.md +++ b/.claude/skills/databricks-model-serving/SKILL.md @@ -29,7 +29,8 @@ ALWAYS use exact endpoint names from this table. NEVER guess or abbreviate. | Endpoint Name | Provider | Notes | |--------------|----------|-------| -| `databricks-gpt-5-2` | OpenAI | Latest GPT, 400K context | +| `databricks-gpt-5-3-codex` | OpenAI | Latest GPT Codex, 400K context | +| `databricks-gpt-5-2` | OpenAI | GPT 5.2, 400K context | | `databricks-gpt-5-1` | OpenAI | Instant + Thinking modes | | `databricks-gpt-5-1-codex-max` | OpenAI | Code-specialized (high perf) | | `databricks-gpt-5-1-codex-mini` | OpenAI | Code-specialized (cost-opt) | diff --git a/README.md b/README.md index 129174cf..87004caa 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ This template repo opens that vision up for every Databricks user — no IDE set | `DATABRICKS_TOKEN` | No | Optional. If not set, the app prompts for a token on first session. Auto-rotated every 10 minutes | | `HOME` | Yes | Set to `/app/python/source_code` in app.yaml | | `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | -| `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-2`) | +| `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-3-codex`) | | `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | | `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL override. Auto-discovered from `DATABRICKS_WORKSPACE_ID` if unset | diff --git a/app.yaml b/app.yaml index 9bf75940..e6bb8cde 100644 --- a/app.yaml +++ b/app.yaml @@ -9,6 +9,6 @@ env: - name: GEMINI_MODEL value: databricks-gemini-3-1-pro - name: CODEX_MODEL - value: databricks-gpt-5-2 + value: databricks-gpt-5-3-codex - name: CLAUDE_CODE_DISABLE_AUTO_MEMORY value: 0 diff --git a/docs/deployment.md b/docs/deployment.md index c7196b59..09959da1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -68,7 +68,7 @@ databricks apps deploy \ | `DATABRICKS_TOKEN` | No | Optional. If not set, the app prompts for a token on first session. Auto-rotated every 10 minutes | | `HOME` | Yes | Set to `/app/python/source_code` in app.yaml | | `ANTHROPIC_MODEL` | No | Claude model name (default: `databricks-claude-opus-4-6`) | -| `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-2`) | +| `CODEX_MODEL` | No | Codex model name (default: `databricks-gpt-5-3-codex`) | | `GEMINI_MODEL` | No | Gemini model name (default: `databricks-gemini-3-1-pro`) | | `DATABRICKS_GATEWAY_HOST` | No | AI Gateway URL override. Auto-discovered from `DATABRICKS_WORKSPACE_ID` if unset. Falls back to direct model serving if neither is available | diff --git a/pyproject.toml b/pyproject.toml index eedde9fb..5a6b6cae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coda" -version = "0.16.6" +version = "0.16.7" description = "CoDA - Coding Agents on Databricks Apps" requires-python = ">=3.10" dependencies = [ diff --git a/setup_codex.py b/setup_codex.py index 6f5238ec..a0f3b72a 100644 --- a/setup_codex.py +++ b/setup_codex.py @@ -22,7 +22,7 @@ host = os.environ.get("DATABRICKS_HOST", "") token = os.environ.get("DATABRICKS_TOKEN", "") -codex_model = os.environ.get("CODEX_MODEL", "databricks-gpt-5-2") +codex_model = os.environ.get("CODEX_MODEL", "databricks-gpt-5-3-codex") # 1. Install Codex CLI into ~/.local/bin (always, even without token) local_bin = home / ".local" / "bin" diff --git a/setup_opencode.py b/setup_opencode.py index 0d791af1..a0ef9c70 100644 --- a/setup_opencode.py +++ b/setup_opencode.py @@ -152,8 +152,8 @@ "compatibility": "compatible" }, "models": { - "databricks-gpt-5-2-codex": { - "name": "GPT 5.2 Codex (Databricks)", + "databricks-gpt-5-3-codex": { + "name": "GPT 5.3 Codex (Databricks)", "limit": { "context": 200000, "output": 16384 @@ -285,6 +285,6 @@ print(f"\nOpenCode ready! Default model: {anthropic_model}") print(" opencode # Start OpenCode TUI") if gateway_host: - print(" opencode -m databricks-openai/databricks-gpt-5-2-codex # Use GPT 5.2 Codex") + print(" opencode -m databricks-openai/databricks-gpt-5-3-codex # Use GPT 5.3 Codex") print(" opencode -m databricks/databricks-gemini-2-5-flash # Use Gemini") print(f" opencode -m databricks/{anthropic_model} # Use Claude (default)") From 49643f4ce108f40a3cc93f0b0c7c5fd3afb53cb6 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Wed, 8 Apr 2026 17:52:01 -0400 Subject: [PATCH 168/382] chore: sync requirements.txt and lockfile with pyproject.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recompile requirements.txt from pyproject.toml (claude-agent-sdk 0.1.50 → 0.1.53, full dependency tree). Regenerate requirements.lock with hashes. Fixes dependency audit CI failure. --- requirements.lock | 648 ++++++++++++++++++++++++++++------------------ requirements.txt | 205 ++++++++++++++- 2 files changed, 598 insertions(+), 255 deletions(-) diff --git a/requirements.lock b/requirements.lock index 9d5bd67a..8a00b772 100644 --- a/requirements.lock +++ b/requirements.lock @@ -3,40 +3,50 @@ annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 - # via pydantic -anyio==4.12.1 \ - --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ - --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c # via + # -r requirements.txt + # pydantic +anyio==4.13.0 \ + --hash=sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708 \ + --hash=sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc + # via + # -r requirements.txt # claude-agent-sdk # httpx # mcp # sse-starlette # starlette -attrs==25.4.0 \ - --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ - --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 # via + # -r requirements.txt # jsonschema # referencing bidict==0.23.1 \ --hash=sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71 \ --hash=sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5 - # via python-socketio + # via + # -r requirements.txt + # python-socketio blinker==1.9.0 \ --hash=sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf \ --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc # via + # -r requirements.txt # flask # flask-socketio cachetools==7.0.5 \ --hash=sha256:0cd042c24377200c1dcd225f8b7b12b0ca53cc2c961b43757e774ebe190fd990 \ --hash=sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114 - # via mlflow-tracing + # via + # -r requirements.txt + # mlflow-tracing certifi==2026.2.25 \ --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 # via + # -r requirements.txt # httpcore # httpx # requests @@ -125,122 +135,142 @@ cffi==2.0.0 \ --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf - # via cryptography -charset-normalizer==3.4.5 \ - --hash=sha256:014837af6fabf57121b6254fa8ade10dceabc3528b27b721a64bbc7b8b1d4eb4 \ - --hash=sha256:01a1ed54b953303ca7e310fafe0fe347aab348bd81834a0bcd602eb538f89d66 \ - --hash=sha256:0294916d6ccf2d069727d65973c3a1ca477d68708db25fd758dd28b0827cff54 \ - --hash=sha256:02a9d1b01c1e12c27883b0c9349e0bcd9ae92e727ff1a277207e1a262b1cbf05 \ - --hash=sha256:036c079aa08a6a592b82487f97c60b439428320ed1b2ea0b3912e99d30c77765 \ - --hash=sha256:039215608ac7b358c4da0191d10fc76868567fbf276d54c14721bdedeb6de064 \ - --hash=sha256:0625665e4ebdddb553ab185de5db7054393af8879fb0c87bd5690d14379d6819 \ - --hash=sha256:0a45e504f5e1be0bd385935a8e1507c442349ca36f511a47057a71c9d1d6ea9e \ - --hash=sha256:0b362bcd27819f9c07cbf23db4e0e8cd4b44c5ecd900c2ff907b2b92274a7412 \ - --hash=sha256:0c300cefd9b0970381a46394902cd18eaf2aa00163f999590ace991989dcd0fc \ - --hash=sha256:1088345bcc93c58d8d8f3d783eca4a6e7a7752bbff26c3eee7e73c597c191c2e \ - --hash=sha256:10b473fc8dca1c3ad8559985794815f06ca3fc71942c969129070f2c3cdf7281 \ - --hash=sha256:131716d6786ad5e3dc542f5cc6f397ba3339dc0fb87f87ac30e550e8987756af \ - --hash=sha256:14498a429321de554b140013142abe7608f9d8ccc04d7baf2ad60498374aefa2 \ - --hash=sha256:149ec69866c3d6c2fb6f758dbc014ecb09f30b35a5ca90b6a8a2d4e54e18fdfe \ - --hash=sha256:165c7b21d19365464e8f70e5ce5e12524c58b48c78c1f5a57524603c1ab003f8 \ - --hash=sha256:1827734a5b308b65ac54e86a618de66f935a4f63a8a462ff1e19a6788d6c2262 \ - --hash=sha256:19092dde50335accf365cce21998a1c6dd8eafd42c7b226eb54b2747cdce2fac \ - --hash=sha256:1a374cc0b88aa710e8865dc1bd6edb3743c59f27830f0293ab101e4cf3ce9f85 \ - --hash=sha256:1d1401945cb77787dbd3af2446ff2d75912327c4c3a1526ab7955ecf8600687c \ - --hash=sha256:1f2da5cbb9becfcd607757a169e38fb82aa5fd86fae6653dea716e7b613fe2cf \ - --hash=sha256:259cd1ca995ad525f638e131dbcc2353a586564c038fc548a3fe450a91882139 \ - --hash=sha256:2820a98460c83663dd8ec015d9ddfd1e4879f12e06bb7d0500f044fb477d2770 \ - --hash=sha256:28269983f25a4da0425743d0d257a2d6921ea7d9b83599d4039486ec5b9f911d \ - --hash=sha256:2b970382e4a36bed897c19f310f31d7d13489c11b4f468ddfba42d41cddfb918 \ - --hash=sha256:2da4eedcb6338e2321e831a0165759c0c620e37f8cd044a263ff67493be8ffb3 \ - --hash=sha256:30987f4a8ed169983f93e1be8ffeea5214a779e27ed0b059835c7afe96550ad7 \ - --hash=sha256:30a2b1a48478c3428d047ed9690d57c23038dac838a87ad624c85c0a78ebeb39 \ - --hash=sha256:340810d34ef83af92148e96e3e44cb2d3f910d2bf95e5618a5c467d9f102231d \ - --hash=sha256:3f64c6bf8f32f9133b668c7f7a7cbdbc453412bc95ecdbd157f3b1e377a92990 \ - --hash=sha256:4167a621a9a1a986c73777dbc15d4b5eac8ac5c10393374109a343d4013ec765 \ - --hash=sha256:4354e401eb6dab9aed3c7b4030514328a6c748d05e1c3e19175008ca7de84fb1 \ - --hash=sha256:4481e6da1830c8a1cc0b746b47f603b653dadb690bcd851d039ffaefe70533aa \ - --hash=sha256:4b8551b6e6531e156db71193771c93bda78ffc4d1e6372517fe58ad3b91e4659 \ - --hash=sha256:4cd966c2559f501c6fd69294d082c2934c8dd4719deb32c22961a5ac6db0df1d \ - --hash=sha256:50bcbca6603c06a1dcc7b056ed45c37715fb5d2768feb3bcd37d2313c587a5b9 \ - --hash=sha256:530beedcec9b6e027e7a4b6ce26eed36678aa39e17da85e6e03d7bd9e8e9d7c9 \ - --hash=sha256:568e3c34b58422075a1b49575a6abc616d9751b4d61b23f712e12ebb78fe47b2 \ - --hash=sha256:573ef5814c4b7c0d59a7710aa920eaaaef383bd71626aa420fba27b5cab92e8d \ - --hash=sha256:58ad8270cfa5d4bef1bc85bd387217e14ff154d6630e976c6f56f9a040757475 \ - --hash=sha256:597d10dec876923e5c59e48dbd366e852eacb2b806029491d307daea6b917d7c \ - --hash=sha256:5bcb3227c3d9aaf73eaaab1db7ccd80a8995c509ee9941e2aae060ca6e4e5d81 \ - --hash=sha256:5cffde4032a197bd3b42fd0b9509ec60fb70918d6970e4cc773f20fc9180ca67 \ - --hash=sha256:5fea359734b140d0d6741189fea5478c6091b54ffc69d7ce119e0a05637d8c99 \ - --hash=sha256:60d68e820af339df4ae8358c7a2e7596badeb61e544438e489035f9fbf3246a5 \ - --hash=sha256:610f72c0ee565dfb8ae1241b666119582fdbfe7c0975c175be719f940e110694 \ - --hash=sha256:65a126fb4b070d05340a84fc709dd9e7c75d9b063b610ece8a60197a291d0adf \ - --hash=sha256:65b3c403a5b6b8034b655e7385de4f72b7b244869a22b32d4030b99a60593eca \ - --hash=sha256:66dee73039277eb35380d1b82cccc69cc82b13a66f9f4a18da32d573acf02b7c \ - --hash=sha256:708c7acde173eedd4bfa4028484426ba689d2103b28588c513b9db2cd5ecde9c \ - --hash=sha256:728c6a963dfab66ef865f49286e45239384249672cd598576765acc2a640a636 \ - --hash=sha256:754f96058e61a5e22e91483f823e07df16416ce76afa4ebf306f8e1d1296d43f \ - --hash=sha256:75dfd1afe0b1647449e852f4fb428195a7ed0588947218f7ba929f6538487f02 \ - --hash=sha256:75ee9c1cce2911581a70a3c0919d8bccf5b1cbc9b0e5171400ec736b4b569497 \ - --hash=sha256:76a9d0de4d0eab387822e7b35d8f89367dd237c72e82ab42b9f7bf5e15ada00f \ - --hash=sha256:77be992288f720306ab4108fe5c74797de327f3248368dfc7e1a916d6ed9e5a2 \ - --hash=sha256:7ad83b8f9379176c841f8865884f3514d905bcd2a9a3b210eaa446e7d2223e4d \ - --hash=sha256:8197abe5ca1ffb7d91e78360f915eef5addff270f8a71c1fc5be24a56f3e4873 \ - --hash=sha256:82cc7c2ad42faec8b574351f8bc2a0c049043893853317bd9bb309f5aba6cb5a \ - --hash=sha256:8a28afb04baa55abf26df544e3e5c6534245d3daa5178bc4a8eeb48202060d0e \ - --hash=sha256:8b78d8a609a4b82c273257ee9d631ded7fac0d875bdcdccc109f3ee8328cfcb1 \ - --hash=sha256:8ce11cd4d62d11166f2b441e30ace226c19a3899a7cf0796f668fba49a9fb123 \ - --hash=sha256:8fff79bf5978c693c9b1a4d71e4a94fddfb5fe744eb062a318e15f4a2f63a550 \ - --hash=sha256:92263f7eca2f4af326cd20de8d16728d2602f7cfea02e790dcde9d83c365d7cc \ - --hash=sha256:93b3b2cc5cf1b8743660ce77a4f45f3f6d1172068207c1defc779a36eea6bb36 \ - --hash=sha256:95adae7b6c42a6c5b5b559b1a99149f090a57128155daeea91732c8d970d8644 \ - --hash=sha256:97ab7787092eb9b50fb47fa04f24c75b768a606af1bcba1957f07f128a7219e4 \ - --hash=sha256:9db5e3fcdcee89a78c04dffb3fe33c79f77bd741a624946db2591c81b2fc85b0 \ - --hash=sha256:a118e2e0b5ae6b0120d5efa5f866e58f2bb826067a646431da4d6a2bdae7950e \ - --hash=sha256:a2aecdb364b8a1802afdc7f9327d55dad5366bc97d8502d0f5854e50712dbc5f \ - --hash=sha256:a66aa5022bf81ab4b1bebfb009db4fd68e0c6d4307a1ce5ef6a26e5878dfc9e4 \ - --hash=sha256:a68766a3c58fde7f9aaa22b3786276f62ab2f594efb02d0a1421b6282e852e98 \ - --hash=sha256:aa2f963b4da26daf46231d9b9e0e2c9408a751f8f0d0f44d2de56d3caf51d294 \ - --hash=sha256:aa92ec1102eaff840ccd1021478af176a831f1bccb08e526ce844b7ddda85c22 \ - --hash=sha256:ac59c15e3f1465f722607800c68713f9fbc2f672b9eb649fe831da4019ae9b23 \ - --hash=sha256:ae8b03427410731469c4033934cf473426faff3e04b69d2dfb64a4281a3719f8 \ - --hash=sha256:afca7f78067dd27c2b848f1b234623d26b87529296c6c5652168cc1954f2f3b2 \ - --hash=sha256:b2d37d78297b39a9eb9eb92c0f6df98c706467282055419df141389b23f93362 \ - --hash=sha256:b3e71afc578b98512bfe7bdb822dd6bc57d4b0093b4b6e5487c1e96ad4ace242 \ - --hash=sha256:ba20bdf69bd127f66d0174d6f2a93e69045e0b4036dc1ca78e091bcc765830c4 \ - --hash=sha256:c108f8619e504140569ee7de3f97d234f0fbae338a7f9f360455071ef9855a95 \ - --hash=sha256:c23eb3263356d94858655b3e63f85ac5d50970c6e8febcdde7830209139cc37d \ - --hash=sha256:c5af897b45fa606b12464ccbe0014bbf8c09191e0a66aab6aa9d5cf6e77e0c94 \ - --hash=sha256:c7a80a9242963416bd81f99349d5f3fce1843c303bd404f204918b6d75a75fd6 \ - --hash=sha256:c7e84e0c0005e3bdc1a9211cd4e62c78ba80bc37b2365ef4410cd2007a9047f2 \ - --hash=sha256:cace89841c0599d736d3d74a27bc5821288bb47c5441923277afc6059d7fbcb4 \ - --hash=sha256:cd2d0f0ec9aa977a27731a3209ebbcacebebaf41f902bd453a928bfd281cf7f8 \ - --hash=sha256:d01de5e768328646e6a3fa9e562706f8f6641708c115c62588aef2b941a4f88e \ - --hash=sha256:d1028de43596a315e2720a9849ee79007ab742c06ad8b45a50db8cdb7ed4a82a \ - --hash=sha256:d27ce22ec453564770d29d03a9506d449efbb9fa13c00842262b2f6801c48cce \ - --hash=sha256:d29dd9c016f2078b43d0c357511e87eee5b05108f3dd603423cb389b89813969 \ - --hash=sha256:d31f0d1671e1534e395f9eb84a68e0fb670e1edb1fe819a9d7f564ae3bc4e53f \ - --hash=sha256:d4eb8ac7469b2a5d64b5b8c04f84d8bf3ad340f4514b98523805cbf46e3b3923 \ - --hash=sha256:d5e52d127045d6ae01a1e821acfad2f3a1866c54d0e837828538fabe8d9d1bd6 \ - --hash=sha256:d77f97e515688bd615c1d1f795d540f32542d514242067adcb8ef532504cb9ee \ - --hash=sha256:d8ed79b8f6372ca4254955005830fd61c1ccdd8c0fac6603e2c145c61dd95db6 \ - --hash=sha256:dc57a0baa3eeedd99fafaef7511b5a6ef4581494e8168ee086031744e2679467 \ - --hash=sha256:e09f671a54ce70b79a1fc1dc6da3072b7ef7251fadb894ed92d9aa8218465a5f \ - --hash=sha256:e22d1059b951e7ae7c20ef6b06afd10fb95e3c41bf3c4fbc874dba113321c193 \ - --hash=sha256:e37bd100d2c5d3ba35db9c7c5ba5a9228cbcffe5c4778dc824b164e5257813d7 \ - --hash=sha256:e51ae7d81c825761d941962450f50d041db028b7278e7b08930b4541b3e45cb9 \ - --hash=sha256:e545b51da9f9af5c67815ca0eb40676c0f016d0b0381c86f20451e35696c5f95 \ - --hash=sha256:e6302ca4ae283deb0af68d2fbf467474b8b6aedcd3dab4db187e07f94c109763 \ - --hash=sha256:e71bbb595973622b817c042bd943c3f3667e9c9983ce3d205f973f486fec98a7 \ - --hash=sha256:ec56a2266f32bc06ed3c3e2a8f58417ce02f7e0356edc89786e52db13c593c98 \ - --hash=sha256:ed1a9a204f317ef879b32f9af507d47e49cd5e7f8e8d5d96358c98373314fc60 \ - --hash=sha256:ed97c282ee4f994ef814042423a529df9497e3c666dca19be1d4cd1129dc7ade \ - --hash=sha256:ed98364e1c262cf5f9363c3eca8c2df37024f52a8fa1180a3610014f26eac51c \ - --hash=sha256:ee57b926940ba00bca7ba7041e665cc956e55ef482f851b9b65acb20d867e7a2 \ - --hash=sha256:f1d725b754e967e648046f00c4facc42d414840f5ccc670c5670f59f83693e4f \ - --hash=sha256:f8102ae93c0bc863b1d41ea0f4499c20a83229f52ed870850892df555187154a \ - --hash=sha256:fc1c64934b8faf7584924143eb9db4770bbdb16659626e1a1a4d9efbcb68d947 \ - --hash=sha256:ff95a9283de8a457e6b12989de3f9f5193430f375d64297d323a615ea52cbdb3 - # via requests + # via + # -r requirements.txt + # cryptography +charset-normalizer==3.4.6 \ + --hash=sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e \ + --hash=sha256:0c173ce3a681f309f31b87125fecec7a5d1347261ea11ebbb856fa6006b23c8c \ + --hash=sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5 \ + --hash=sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815 \ + --hash=sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f \ + --hash=sha256:150b8ce8e830eb7ccb029ec9ca36022f756986aaaa7956aad6d9ec90089338c0 \ + --hash=sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484 \ + --hash=sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407 \ + --hash=sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6 \ + --hash=sha256:1cf0a70018692f85172348fe06d3a4b63f94ecb055e13a00c644d368eb82e5b8 \ + --hash=sha256:1ed80ff870ca6de33f4d953fda4d55654b9a2b340ff39ab32fa3adbcd718f264 \ + --hash=sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815 \ + --hash=sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2 \ + --hash=sha256:259695e2ccc253feb2a016303543d691825e920917e31f894ca1a687982b1de4 \ + --hash=sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579 \ + --hash=sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f \ + --hash=sha256:2bd9d128ef93637a5d7a6af25363cf5dec3fa21cf80e68055aad627f280e8afa \ + --hash=sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95 \ + --hash=sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab \ + --hash=sha256:2f7fdd9b6e6c529d6a2501a2d36b240109e78a8ceaef5687cfcfa2bbe671d297 \ + --hash=sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a \ + --hash=sha256:31215157227939b4fb3d740cd23fe27be0439afef67b785a1eb78a3ae69cba9e \ + --hash=sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84 \ + --hash=sha256:3516bbb8d42169de9e61b8520cbeeeb716f12f4ecfe3fd30a9919aa16c806ca8 \ + --hash=sha256:3778fd7d7cd04ae8f54651f4a7a0bd6e39a0cf20f801720a4c21d80e9b7ad6b0 \ + --hash=sha256:39f5068d35621da2881271e5c3205125cc456f54e9030d3f723288c873a71bf9 \ + --hash=sha256:404a1e552cf5b675a87f0651f8b79f5f1e6fd100ee88dc612f89aa16abd4486f \ + --hash=sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1 \ + --hash=sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843 \ + --hash=sha256:4482481cb0572180b6fd976a4d5c72a30263e98564da68b86ec91f0fe35e8565 \ + --hash=sha256:461598cd852bfa5a61b09cae2b1c02e2efcd166ee5516e243d540ac24bfa68a7 \ + --hash=sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c \ + --hash=sha256:48696db7f18afb80a068821504296eb0787d9ce239b91ca15059d1d3eaacf13b \ + --hash=sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7 \ + --hash=sha256:4d1d02209e06550bdaef34af58e041ad71b88e624f5d825519da3a3308e22687 \ + --hash=sha256:4f41da960b196ea355357285ad1316a00099f22d0929fe168343b99b254729c9 \ + --hash=sha256:517ad0e93394ac532745129ceabdf2696b609ec9f87863d337140317ebce1c14 \ + --hash=sha256:51fb3c322c81d20567019778cb5a4a6f2dc1c200b886bc0d636238e364848c89 \ + --hash=sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f \ + --hash=sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0 \ + --hash=sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9 \ + --hash=sha256:54fae94be3d75f3e573c9a1b5402dc593de19377013c9a0e4285e3d402dd3a2a \ + --hash=sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389 \ + --hash=sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0 \ + --hash=sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30 \ + --hash=sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd \ + --hash=sha256:5feb91325bbceade6afab43eb3b508c63ee53579fe896c77137ded51c6b6958e \ + --hash=sha256:60c74963d8350241a79cb8feea80e54d518f72c26db618862a8f53e5023deaf9 \ + --hash=sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc \ + --hash=sha256:659a1e1b500fac8f2779dd9e1570464e012f43e580371470b45277a27baa7532 \ + --hash=sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d \ + --hash=sha256:69dd852c2f0ad631b8b60cfbe25a28c0058a894de5abb566619c205ce0550eae \ + --hash=sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2 \ + --hash=sha256:71be7e0e01753a89cf024abf7ecb6bca2c81738ead80d43004d9b5e3f1244e64 \ + --hash=sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f \ + --hash=sha256:74a2e659c7ecbc73562e2a15e05039f1e22c75b7c7618b4b574a3ea9118d1557 \ + --hash=sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e \ + --hash=sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff \ + --hash=sha256:7a6967aaf043bceabab5412ed6bd6bd26603dae84d5cb75bf8d9a74a4959d398 \ + --hash=sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db \ + --hash=sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a \ + --hash=sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43 \ + --hash=sha256:802168e03fba8bbc5ce0d866d589e4b1ca751d06edee69f7f3a19c5a9fe6b597 \ + --hash=sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c \ + --hash=sha256:82060f995ab5003a2d6e0f4ad29065b7672b6593c8c63559beefe5b443242c3e \ + --hash=sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2 \ + --hash=sha256:8761ac29b6c81574724322a554605608a9960769ea83d2c73e396f3df896ad54 \ + --hash=sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e \ + --hash=sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4 \ + --hash=sha256:8bc5f0687d796c05b1e28ab0d38a50e6309906ee09375dd3aff6a9c09dd6e8f4 \ + --hash=sha256:8bea55c4eef25b0b19a0337dc4e3f9a15b00d569c77211fa8cde38684f234fb7 \ + --hash=sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6 \ + --hash=sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5 \ + --hash=sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194 \ + --hash=sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69 \ + --hash=sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f \ + --hash=sha256:97d0235baafca5f2b09cf332cc275f021e694e8362c6bb9c96fc9a0eb74fc316 \ + --hash=sha256:9ca4c0b502ab399ef89248a2c84c54954f77a070f28e546a85e91da627d1301e \ + --hash=sha256:9cc4fc6c196d6a8b76629a70ddfcd4635a6898756e2d9cac5565cf0654605d73 \ + --hash=sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8 \ + --hash=sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923 \ + --hash=sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88 \ + --hash=sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f \ + --hash=sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21 \ + --hash=sha256:a9e68c9d88823b274cf1e72f28cb5dc89c990edf430b0bfd3e2fb0785bfeabf4 \ + --hash=sha256:aa9cccf4a44b9b62d8ba8b4dd06c649ba683e4bf04eea606d2e94cfc2d6ff4d6 \ + --hash=sha256:ab30e5e3e706e3063bc6de96b118688cb10396b70bb9864a430f67df98c61ecc \ + --hash=sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2 \ + --hash=sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866 \ + --hash=sha256:b35b200d6a71b9839a46b9b7fff66b6638bb52fc9658aa58796b0326595d3021 \ + --hash=sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2 \ + --hash=sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d \ + --hash=sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8 \ + --hash=sha256:bc72863f4d9aba2e8fd9085e63548a324ba706d2ea2c83b260da08a59b9482de \ + --hash=sha256:bf625105bb9eef28a56a943fec8c8a98aeb80e7d7db99bd3c388137e6eb2d237 \ + --hash=sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4 \ + --hash=sha256:c45a03a4c69820a399f1dda9e1d8fbf3562eda46e7720458180302021b08f778 \ + --hash=sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb \ + --hash=sha256:c907cdc8109f6c619e6254212e794d6548373cc40e1ec75e6e3823d9135d29cc \ + --hash=sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602 \ + --hash=sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4 \ + --hash=sha256:d08ec48f0a1c48d75d0356cea971921848fb620fdeba805b28f937e90691209f \ + --hash=sha256:d1a2ee9c1499fc8f86f4521f27a973c914b211ffa87322f4ee33bb35392da2c5 \ + --hash=sha256:d5f5d1e9def3405f60e3ca8232d56f35c98fb7bf581efcc60051ebf53cb8b611 \ + --hash=sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8 \ + --hash=sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf \ + --hash=sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d \ + --hash=sha256:dad6e0f2e481fffdcf776d10ebee25e0ef89f16d691f1e5dee4b586375fdc64b \ + --hash=sha256:dda86aba335c902b6149a02a55b38e96287157e609200811837678214ba2b1db \ + --hash=sha256:df01808ee470038c3f8dc4f48620df7225c49c2d6639e38f96e6d6ac6e6f7b0e \ + --hash=sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077 \ + --hash=sha256:e25369dc110d58ddf29b949377a93e0716d72a24f62bad72b2b39f155949c1fd \ + --hash=sha256:e3c701e954abf6fc03a49f7c579cc80c2c6cc52525340ca3186c41d3f33482ef \ + --hash=sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e \ + --hash=sha256:e68c14b04827dd76dcbd1aeea9e604e3e4b78322d8faf2f8132c7138efa340a8 \ + --hash=sha256:e8aeb10fcbe92767f0fa69ad5a72deca50d0dca07fbde97848997d778a50c9fe \ + --hash=sha256:e985a16ff513596f217cee86c21371b8cd011c0f6f056d0920aa2d926c544058 \ + --hash=sha256:ecbbd45615a6885fe3240eb9db73b9e62518b611850fdf8ab08bd56de7ad2b17 \ + --hash=sha256:ee4ec14bc1680d6b0afab9aea2ef27e26d2024f18b24a2d7155a52b60da7e833 \ + --hash=sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421 \ + --hash=sha256:f0cdaecd4c953bfae0b6bb64910aaaca5a424ad9c72d85cb88417bb9814f7550 \ + --hash=sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff \ + --hash=sha256:f50498891691e0864dc3da965f340fada0771f6142a378083dc4608f4ea513e2 \ + --hash=sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc \ + --hash=sha256:f61aa92e4aad0be58eb6eb4e0c21acf32cf8065f4b2cae5665da756c4ceef982 \ + --hash=sha256:f6e4333fb15c83f7d1482a76d45a0818897b3d33f00efd215528ff7c51b8e35d \ + --hash=sha256:f820f24b09e3e779fe84c3c456cb4108a7aa639b0d1f02c28046e11bfcd088ed \ + --hash=sha256:f98059e4fcd3e3e4e2d632b7cf81c2faae96c43c60b569e9c621468082f1d104 \ + --hash=sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659 + # via + # -r requirements.txt + # requests claude-agent-sdk==0.1.50 \ --hash=sha256:2e44caf3e5bce56e26a18158acf3e1c2c2784cf8fa15e425afe92816c987eb1a \ --hash=sha256:44e75b9d076bd6030742729f99eb38777b80f052b22338d0a028d8190fc59e52 \ @@ -253,12 +283,63 @@ click==8.3.1 \ --hash=sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a \ --hash=sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6 # via + # -r requirements.txt # flask # flask-socketio # uvicorn -cryptography==46.0.6 +cryptography==46.0.6 \ + --hash=sha256:02fad249cb0e090b574e30b276a3da6a149e04ee2f049725b1f69e7b8351ec70 \ + --hash=sha256:063b67749f338ca9c5a0b7fe438a52c25f9526b851e24e6c9310e7195aad3b4d \ + --hash=sha256:12cae594e9473bca1a7aceb90536060643128bb274fcea0fc459ab90f7d1ae7a \ + --hash=sha256:12f0fa16cc247b13c43d56d7b35287ff1569b5b1f4c5e87e92cc4fcc00cd10c0 \ + --hash=sha256:22259338084d6ae497a19bae5d4c66b7ca1387d3264d1c2c0e72d9e9b6a77b97 \ + --hash=sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30 \ + --hash=sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759 \ + --hash=sha256:2b417edbe8877cda9022dde3a008e2deb50be9c407eef034aeeb3a8b11d9db3c \ + --hash=sha256:2ea0f37e9a9cf0df2952893ad145fd9627d326a59daec9b0802480fa3bcd2ead \ + --hash=sha256:2ef9e69886cbb137c2aef9772c2e7138dc581fad4fcbcf13cc181eb5a3ab6275 \ + --hash=sha256:341359d6c9e68834e204ceaf25936dffeafea3829ab80e9503860dcc4f4dac58 \ + --hash=sha256:380343e0653b1c9d7e1f55b52aaa2dbb2fdf2730088d48c43ca1c7c0abb7cc2f \ + --hash=sha256:3c21d92ed15e9cfc6eb64c1f5a0326db22ca9c2566ca46d845119b45b4400361 \ + --hash=sha256:3dfa6567f2e9e4c5dceb8ccb5a708158a2a871052fa75c8b78cb0977063f1507 \ + --hash=sha256:456b3215172aeefb9284550b162801d62f5f264a081049a3e94307fe20792cfa \ + --hash=sha256:4668298aef7cddeaf5c6ecc244c2302a2b8e40f384255505c22875eebb47888b \ + --hash=sha256:50575a76e2951fe7dbd1f56d181f8c5ceeeb075e9ff88e7ad997d2f42af06e7b \ + --hash=sha256:639301950939d844a9e1c4464d7e07f902fe9a7f6b215bb0d4f28584729935d8 \ + --hash=sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8 \ + --hash=sha256:6617f67b1606dfd9fe4dbfa354a9508d4a6d37afe30306fe6c101b7ce3274b72 \ + --hash=sha256:67177e8a9f421aa2d3a170c3e56eca4e0128883cf52a071a7cbf53297f18b175 \ + --hash=sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e \ + --hash=sha256:6739d56300662c468fddb0e5e291f9b4d084bead381667b9e654c7dd81705124 \ + --hash=sha256:69cf0056d6947edc6e6760e5f17afe4bea06b56a9ac8a06de9d2bd6b532d4f3a \ + --hash=sha256:760997a4b950ff00d418398ad73fbc91aa2894b5c1db7ccb45b4f68b42a63b3c \ + --hash=sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f \ + --hash=sha256:7e6142674f2a9291463e5e150090b95a8519b2fb6e6aaec8917dd8d094ce750d \ + --hash=sha256:7f417f034f91dcec1cb6c5c35b07cdbb2ef262557f701b4ecd803ee8cefed4f4 \ + --hash=sha256:7f6690b6c55e9c5332c0b59b9c8a3fb232ebf059094c17f9019a51e9827df91c \ + --hash=sha256:8927ccfbe967c7df312ade694f987e7e9e22b2425976ddbf28271d7e58845290 \ + --hash=sha256:8ce35b77aaf02f3b59c90b2c8a05c73bac12cea5b4e8f3fbece1f5fddea5f0ca \ + --hash=sha256:8e7304c4f4e9490e11efe56af6713983460ee0780f16c63f219984dab3af9d2d \ + --hash=sha256:90e5f0a7b3be5f40c3a0a0eafb32c681d8d2c181fc2a1bdabe9b3f611d9f6b1a \ + --hash=sha256:97c8115b27e19e592a05c45d0dd89c57f81f841cc9880e353e0d3bf25b2139ed \ + --hash=sha256:9a693028b9cbe51b5a1136232ee8f2bc242e4e19d456ded3fa7c86e43c713b4a \ + --hash=sha256:9a9c42a2723999a710445bc0d974e345c32adfd8d2fac6d8a251fa829ad31cfb \ + --hash=sha256:a3e84d5ec9ba01f8fd03802b2147ba77f0c8f2617b2aff254cedd551844209c8 \ + --hash=sha256:aad75154a7ac9039936d50cf431719a2f8d4ed3d3c277ac03f3339ded1a5e707 \ + --hash=sha256:b12c6b1e1651e42ab5de8b1e00dc3b6354fdfd778e7fa60541ddacc27cd21410 \ + --hash=sha256:b928a3ca837c77a10e81a814a693f2295200adb3352395fad024559b7be7a736 \ + --hash=sha256:bcb87663e1f7b075e48c3be3ecb5f0b46c8fc50b50a97cf264e7f60242dca3f2 \ + --hash=sha256:c797e2517cb7880f8297e2c0f43bb910e91381339336f75d2c1c2cbf811b70b4 \ + --hash=sha256:c89eb37fae9216985d8734c1afd172ba4927f5a05cfd9bf0e4863c6d5465b013 \ + --hash=sha256:cdcd3edcbc5d55757e5f5f3d330dd00007ae463a7e7aa5bf132d1f22a4b62b19 \ + --hash=sha256:d24c13369e856b94892a89ddf70b332e0b70ad4a5c43cf3e9cb71d6d7ffa1f7b \ + --hash=sha256:d4e4aadb7fc1f88687f47ca20bb7227981b03afaae69287029da08096853b738 \ + --hash=sha256:d9528b535a6c4f8ff37847144b8986a9a143585f0540fbcb1a98115b543aa463 \ + --hash=sha256:ed3775295fb91f70b4027aeba878d79b3e55c0b3e97eaa4de71f8f23a9f2eb77 \ + --hash=sha256:ed418c37d095aeddf5336898a132fba01091f0ac5844e3e8018506f014b6d2c4 # via # -r requirements.txt + # google-auth # pyjwt databricks-sdk==0.102.0 \ --hash=sha256:75d1253276ee8f3dd5e7b00d62594b7051838435e618f74a8570a6dbd723ec12 \ @@ -276,125 +357,148 @@ flask-socketio==5.6.1 \ --hash=sha256:51a3f71b28b4476c650829607e3a993e076034db6c3cc31f718f0a4b45939d42 \ --hash=sha256:fe5bd995c3ed4da9a98f335d0d830fa1a19d84a64789f6265642a671fdacaeac # via -r requirements.txt -google-auth==2.47.0 \ - --hash=sha256:833229070a9dfee1a353ae9877dcd2dec069a8281a4e72e72f77d4a70ff945da \ - --hash=sha256:c516d68336bfde7cf0da26aab674a36fedcf04b37ac4edd59c597178760c3498 - # via databricks-sdk -googleapis-common-protos==1.73.0 \ - --hash=sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a \ - --hash=sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8 - # via opentelemetry-exporter-otlp-proto-grpc -grpcio==1.78.0 \ - --hash=sha256:082653eecbdf290e6e3e2c276ab2c54b9e7c299e07f4221872380312d8cf395e \ - --hash=sha256:10a9a644b5dd5aec3b82b5b0b90d41c0fa94c85ef42cb42cf78a23291ddb5e7d \ - --hash=sha256:12a771591ae40bc65ba67048fa52ef4f0e6db8279e595fd349f9dfddeef571f9 \ - --hash=sha256:185dea0d5260cbb2d224c507bf2a5444d5abbb1fa3594c1ed7e4c709d5eb8383 \ - --hash=sha256:1afa62af6e23f88629f2b29ec9e52ec7c65a7176c1e0a83292b93c76ca882558 \ - --hash=sha256:2045397e63a7a0ee7957c25f7dbb36ddc110e0cfb418403d110c0a7a68a844e9 \ - --hash=sha256:207db540302c884b8848036b80db352a832b99dfdf41db1eb554c2c2c7800f65 \ - --hash=sha256:271c73e6e5676afe4fc52907686670c7cea22ab2310b76a59b678403ed40d670 \ - --hash=sha256:2777b783f6c13b92bd7b716667452c329eefd646bfb3f2e9dabea2e05dbd34f6 \ - --hash=sha256:2bf5e2e163b356978b23652c4818ce4759d40f4712ee9ec5a83c4be6f8c23a3a \ - --hash=sha256:35eb275bf1751d2ffbd8f57cdbc46058e857cf3971041521b78b7db94bdaf127 \ - --hash=sha256:391e93548644e6b2726f1bb84ed60048d4bcc424ce5e4af0843d28ca0b754fec \ - --hash=sha256:3c586ac70e855c721bda8f548d38c3ca66ac791dc49b66a8281a1f99db85e452 \ - --hash=sha256:3f8904a8165ab21e07e58bf3e30a73f4dffc7a1e0dbc32d51c61b5360d26f43e \ - --hash=sha256:459ab414b35f4496138d0ecd735fed26f1318af5e52cb1efbc82a09f0d5aa911 \ - --hash=sha256:4c5533d03a6cbd7f56acfc9cfb44ea64f63d29091e40e44010d34178d392d7eb \ - --hash=sha256:51b13f9aed9d59ee389ad666b8c2214cc87b5de258fa712f9ab05f922e3896c6 \ - --hash=sha256:5361a0630a7fdb58a6a97638ab70e1dae2893c4d08d7aba64ded28bb9e7a29df \ - --hash=sha256:5397fff416b79e4b284959642a4e95ac4b0f1ece82c9993658e0e477d40551ec \ - --hash=sha256:57bab6deef2f4f1ca76cc04565df38dc5713ae6c17de690721bdf30cb1e0545c \ - --hash=sha256:6092beabe1966a3229f599d7088b38dfc8ffa1608b5b5cdda31e591e6500f856 \ - --hash=sha256:684083fd383e9dc04c794adb838d4faea08b291ce81f64ecd08e4577c7398adf \ - --hash=sha256:735e38e176a88ce41840c21bb49098ab66177c64c82426e24e0082500cc68af5 \ - --hash=sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5 \ - --hash=sha256:748b6138585379c737adc08aeffd21222abbda1a86a0dca2a39682feb9196c20 \ - --hash=sha256:74be1268d1439eaaf552c698cdb11cd594f0c49295ae6bb72c34ee31abbe611b \ - --hash=sha256:7cc47943d524ee0096f973e1081cb8f4f17a4615f2116882a5f1416e4cfe92b5 \ - --hash=sha256:859b13906ce098c0b493af92142ad051bf64c7870fa58a123911c88606714996 \ - --hash=sha256:85f93781028ec63f383f6bc90db785a016319c561cc11151fbb7b34e0d012303 \ - --hash=sha256:86ce2371bfd7f212cf60d8517e5e854475c2c43ce14aa910e136ace72c6db6c1 \ - --hash=sha256:86f85dd7c947baa707078a236288a289044836d4b640962018ceb9cd1f899af5 \ - --hash=sha256:8dfffba826efcf366b1e3ccc37e67afe676f290e13a3b48d31a46739f80a8724 \ - --hash=sha256:8f2ac84905d12918e4e55a16da17939eb63e433dc11b677267c35568aa63fc84 \ - --hash=sha256:94309f498bcc07e5a7d16089ab984d42ad96af1d94b5a4eb966a266d9fcabf68 \ - --hash=sha256:94f95cf5d532d0e717eed4fc1810e8e6eded04621342ec54c89a7c2f14b581bf \ - --hash=sha256:9566fe4ababbb2610c39190791e5b829869351d14369603702e890ef3ad2d06e \ - --hash=sha256:9dca934f24c732750389ce49d638069c3892ad065df86cb465b3fa3012b70c9e \ - --hash=sha256:a9f136fbafe7ccf4ac7e8e0c28b31066e810be52d6e344ef954a3a70234e1702 \ - --hash=sha256:ab399ef5e3cd2a721b1038a0f3021001f19c5ab279f145e1146bb0b9f1b2b12c \ - --hash=sha256:b0c689c02947d636bc7fab3e30cc3a3445cca99c834dfb77cd4a6cabfc1c5597 \ - --hash=sha256:b2342d87af32790f934a79c3112641e7b27d63c261b8b4395350dad43eff1dc7 \ - --hash=sha256:b58f37edab4a3881bc6c9bca52670610e0c9ca14e2ea3cf9debf185b870457fb \ - --hash=sha256:bd8cb8026e5f5b50498a3c4f196f57f9db344dad829ffae16b82e4fdbaea2813 \ - --hash=sha256:be63c88b32e6c0f1429f1398ca5c09bc64b0d80950c8bb7807d7d7fb36fb84c7 \ - --hash=sha256:c3f293fdc675ccba4db5a561048cca627b5e7bd1c8a6973ffedabe7d116e22e2 \ - --hash=sha256:c41bc64626db62e72afec66b0c8a0da76491510015417c127bfc53b2fe6d7f7f \ - --hash=sha256:ce3a90455492bf8bfa38e56fbbe1dbd4f872a3d8eeaf7337dc3b1c8aa28c271b \ - --hash=sha256:ce7599575eeb25c0f4dc1be59cada6219f3b56176f799627f44088b21381a28a \ - --hash=sha256:dce09d6116df20a96acfdbf85e4866258c3758180e8c49845d6ba8248b6d0bbb \ - --hash=sha256:de8cb00d1483a412a06394b8303feec5dcb3b55f81d83aa216dbb6a0b86a94f5 \ - --hash=sha256:df2c8f3141f7cbd112a6ebbd760290b5849cda01884554f7c67acc14e7b1758a \ - --hash=sha256:e87cbc002b6f440482b3519e36e1313eb5443e9e9e73d6a52d43bd2004fcfd8e \ - --hash=sha256:e888474dee2f59ff68130f8a397792d8cb8e17e6b3434339657ba4ee90845a8c \ - --hash=sha256:f12857d24d98441af6a1d5c87442d624411db486f7ba12550b07788f74b67b04 \ - --hash=sha256:f2d4e43ee362adfc05994ed479334d5a451ab7bc3f3fee1b796b8ca66895acb4 \ - --hash=sha256:f3d6379493e18ad4d39537a82371c5281e153e963cecb13f953ebac155756525 \ - --hash=sha256:f8dff3d9777e5d2703a962ee5c286c239bf0ba173877cc68dc02c17d042e29de \ - --hash=sha256:f9ab915a267fc47c7e88c387a3a28325b58c898e23d4995f765728f4e3dedb97 \ - --hash=sha256:fbe6e89c7ffb48518384068321621b2a69cab509f58e40e4399fdd378fa6d074 \ - --hash=sha256:fd5f135b1bd58ab088930b3c613455796dfa0393626a6972663ccdda5b4ac6ce \ - --hash=sha256:ff870aebe9a93a85283837801d35cd5f8814fe2ad01e606861a7fb47c762a2b7 - # via opentelemetry-exporter-otlp-proto-grpc +google-auth==2.49.1 \ + --hash=sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64 \ + --hash=sha256:195ebe3dca18eddd1b3db5edc5189b76c13e96f29e73043b923ebcf3f1a860f7 + # via + # -r requirements.txt + # databricks-sdk +googleapis-common-protos==1.73.1 \ + --hash=sha256:13114f0e9d2391756a0194c3a8131974ed7bffb06086569ba193364af59163b6 \ + --hash=sha256:e51f09eb0a43a8602f5a915870972e6b4a394088415c79d79605a46d8e826ee8 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp-proto-grpc +grpcio==1.80.0 \ + --hash=sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1 \ + --hash=sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9 \ + --hash=sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de \ + --hash=sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab \ + --hash=sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921 \ + --hash=sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957 \ + --hash=sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f \ + --hash=sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257 \ + --hash=sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d \ + --hash=sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05 \ + --hash=sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd \ + --hash=sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e \ + --hash=sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02 \ + --hash=sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae \ + --hash=sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f \ + --hash=sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21 \ + --hash=sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e \ + --hash=sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f \ + --hash=sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1 \ + --hash=sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd \ + --hash=sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069 \ + --hash=sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411 \ + --hash=sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14 \ + --hash=sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9 \ + --hash=sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440 \ + --hash=sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc \ + --hash=sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060 \ + --hash=sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141 \ + --hash=sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6 \ + --hash=sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388 \ + --hash=sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106 \ + --hash=sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140 \ + --hash=sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c \ + --hash=sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f \ + --hash=sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7 \ + --hash=sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0 \ + --hash=sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294 \ + --hash=sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f \ + --hash=sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff \ + --hash=sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f \ + --hash=sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc \ + --hash=sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199 \ + --hash=sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4 \ + --hash=sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2 \ + --hash=sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7 \ + --hash=sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4 \ + --hash=sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a \ + --hash=sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0 \ + --hash=sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193 \ + --hash=sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6 \ + --hash=sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de \ + --hash=sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f \ + --hash=sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58 \ + --hash=sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614 \ + --hash=sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a \ + --hash=sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50 \ + --hash=sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad \ + --hash=sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4 \ + --hash=sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9 \ + --hash=sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2 \ + --hash=sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp-proto-grpc h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via + # -r requirements.txt # httpcore # uvicorn # wsproto httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 - # via httpx + # via + # -r requirements.txt + # httpx httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad - # via mcp + # via + # -r requirements.txt + # mcp httpx-sse==0.4.3 \ --hash=sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc \ --hash=sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d - # via mcp + # via + # -r requirements.txt + # mcp idna==3.11 \ --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 # via + # -r requirements.txt # anyio # httpx # requests importlib-metadata==8.7.1 \ --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \ --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151 - # via opentelemetry-api + # via + # -r requirements.txt + # opentelemetry-api itsdangerous==2.2.0 \ --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \ --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173 - # via flask + # via + # -r requirements.txt + # flask jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via + # -r requirements.txt # flask # flask-socketio jsonschema==4.26.0 \ --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce - # via mcp + # via + # -r requirements.txt + # mcp jsonschema-specifications==2025.9.1 \ --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d - # via jsonschema + # via + # -r requirements.txt + # jsonschema markupsafe==3.0.3 \ --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ @@ -486,13 +590,16 @@ markupsafe==3.0.3 \ --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 # via + # -r requirements.txt # flask # jinja2 # werkzeug mcp==1.26.0 \ --hash=sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca \ --hash=sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66 - # via claude-agent-sdk + # via + # -r requirements.txt + # claude-agent-sdk mlflow-tracing==3.10.1 \ --hash=sha256:649c722cc58d54f1f40559023a6bd6f3f08150c3ce3c3bb27972b3e795890f47 \ --hash=sha256:9e54d63cf776d29bb9e2278d35bf27352b93f7b35c8fe8452e9ba5e2a3c5b78f @@ -501,6 +608,7 @@ opentelemetry-api==1.40.0 \ --hash=sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f \ --hash=sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9 # via + # -r requirements.txt # mlflow-tracing # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-sdk @@ -508,7 +616,9 @@ opentelemetry-api==1.40.0 \ opentelemetry-exporter-otlp-proto-common==1.40.0 \ --hash=sha256:1cbee86a4064790b362a86601ee7934f368b81cd4cc2f2e163902a6e7818a0fa \ --hash=sha256:7081ff453835a82417bf38dccf122c827c3cbc94f2079b03bba02a3165f25149 - # via opentelemetry-exporter-otlp-proto-grpc + # via + # -r requirements.txt + # opentelemetry-exporter-otlp-proto-grpc opentelemetry-exporter-otlp-proto-grpc==1.40.0 \ --hash=sha256:2aa0ca53483fe0cf6405087a7491472b70335bc5c7944378a0a8e72e86995c52 \ --hash=sha256:bd4015183e40b635b3dab8da528b27161ba83bf4ef545776b196f0fb4ec47740 @@ -517,6 +627,7 @@ opentelemetry-proto==1.40.0 \ --hash=sha256:03f639ca129ba513f5819810f5b1f42bcb371391405d99c168fe6937c62febcd \ --hash=sha256:266c4385d88923a23d63e353e9761af0f47a6ed0d486979777fe4de59dc9b25f # via + # -r requirements.txt # mlflow-tracing # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc @@ -524,28 +635,34 @@ opentelemetry-sdk==1.40.0 \ --hash=sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2 \ --hash=sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1 # via + # -r requirements.txt # mlflow-tracing # opentelemetry-exporter-otlp-proto-grpc opentelemetry-semantic-conventions==0.61b0 \ --hash=sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a \ --hash=sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2 - # via opentelemetry-sdk + # via + # -r requirements.txt + # opentelemetry-sdk packaging==26.0 \ --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 - # via mlflow-tracing -protobuf==6.33.5 \ - --hash=sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c \ - --hash=sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02 \ - --hash=sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c \ - --hash=sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd \ - --hash=sha256:8f04fa32763dcdb4973d537d6b54e615cc61108c7cb38fe59310c3192d29510a \ - --hash=sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190 \ - --hash=sha256:a3157e62729aafb8df6da2c03aa5c0937c7266c626ce11a278b6eb7963c4e37c \ - --hash=sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 \ - --hash=sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 \ - --hash=sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b # via + # -r requirements.txt + # mlflow-tracing +protobuf==6.33.6 \ + --hash=sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326 \ + --hash=sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901 \ + --hash=sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3 \ + --hash=sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a \ + --hash=sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135 \ + --hash=sha256:bd56799fb262994b2c2faa1799693c95cc2e22c62f56fb43af311cae45d26f0e \ + --hash=sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3 \ + --hash=sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2 \ + --hash=sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593 \ + --hash=sha256:f443a394af5ed23672bc6c486be138628fbe5c651ccbc536873d7da23d1868cf + # via + # -r requirements.txt # databricks-sdk # googleapis-common-protos # mlflow-tracing @@ -554,20 +671,25 @@ pyasn1==0.6.3 \ --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \ --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde # via + # -r requirements.txt # pyasn1-modules - # rsa pyasn1-modules==0.4.2 \ --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 - # via google-auth + # via + # -r requirements.txt + # google-auth pycparser==3.0 \ --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 - # via cffi + # via + # -r requirements.txt + # cffi pydantic==2.12.5 \ --hash=sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49 \ --hash=sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d # via + # -r requirements.txt # mcp # mlflow-tracing # pydantic-settings @@ -693,35 +815,50 @@ pydantic-core==2.41.5 \ --hash=sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7 \ --hash=sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425 \ --hash=sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52 - # via pydantic + # via + # -r requirements.txt + # pydantic pydantic-settings==2.13.1 \ --hash=sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025 \ --hash=sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237 - # via mcp + # via + # -r requirements.txt + # mcp pyjwt==2.12.1 \ --hash=sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c \ --hash=sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b - # via mcp + # via + # -r requirements.txt + # mcp python-dotenv==1.2.2 \ --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a \ --hash=sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3 - # via pydantic-settings + # via + # -r requirements.txt + # pydantic-settings python-engineio==4.13.1 \ --hash=sha256:0a853fcef52f5b345425d8c2b921ac85023a04dfcf75d7b74696c61e940fd066 \ --hash=sha256:f32ad10589859c11053ad7d9bb3c9695cdf862113bfb0d20bc4d890198287399 - # via python-socketio + # via + # -r requirements.txt + # python-socketio python-multipart==0.0.22 \ --hash=sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155 \ --hash=sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58 - # via mcp + # via + # -r requirements.txt + # mcp python-socketio==5.16.1 \ --hash=sha256:a3eb1702e92aa2f2b5d3ba00261b61f062cce51f1cfb6900bf3ab4d1934d2d35 \ --hash=sha256:f863f98eacce81ceea2e742f6388e10ca3cdd0764be21d30d5196470edf5ea89 - # via flask-socketio + # via + # -r requirements.txt + # flask-socketio referencing==0.37.0 \ --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 \ --hash=sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8 # via + # -r requirements.txt # jsonschema # jsonschema-specifications requests @ git+https://github.com/psf/requests@bc04dfd6dad4cb02cd92f5daa81eb562d280a761 @@ -845,32 +982,33 @@ rpds-py==0.30.0 \ --hash=sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f \ --hash=sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5 # via + # -r requirements.txt # jsonschema # referencing -rsa==4.9.1 \ - --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ - --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 - # via google-auth simple-websocket==1.1.0 \ --hash=sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c \ --hash=sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4 # via # -r requirements.txt # python-engineio -sse-starlette==3.3.2 \ - --hash=sha256:5c3ea3dad425c601236726af2f27689b74494643f57017cafcb6f8c9acfbb862 \ - --hash=sha256:678fca55a1945c734d8472a6cad186a55ab02840b4f6786f5ee8770970579dcd - # via mcp -starlette==0.52.1 \ - --hash=sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74 \ - --hash=sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933 +sse-starlette==3.3.4 \ + --hash=sha256:84bb06e58939a8b38d8341f1bc9792f06c2b53f48c608dd207582b664fc8f3c1 \ + --hash=sha256:aaf92fc067af8a5427192895ac028e947b484ac01edbc3caf00e7e7137c7bef1 + # via + # -r requirements.txt + # mcp +starlette==1.0.0 \ + --hash=sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149 \ + --hash=sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b # via + # -r requirements.txt # mcp # sse-starlette typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via + # -r requirements.txt # anyio # grpcio # mcp @@ -887,28 +1025,38 @@ typing-inspection==0.4.2 \ --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 # via + # -r requirements.txt # mcp # pydantic # pydantic-settings urllib3==2.6.3 \ --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 - # via requests -uvicorn==0.41.0 \ - --hash=sha256:09d11cf7008da33113824ee5a1c6422d89fbc2ff476540d69a34c87fab8b571a \ - --hash=sha256:29e35b1d2c36a04b9e180d4007ede3bcb32a85fbdfd6c6aeb3f26839de088187 - # via mcp -werkzeug==3.1.6 \ - --hash=sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25 \ - --hash=sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131 # via + # -r requirements.txt + # requests +uvicorn==0.42.0 \ + --hash=sha256:96c30f5c7abe6f74ae8900a70e92b85ad6613b745d4879eb9b16ccad15645359 \ + --hash=sha256:9b1f190ce15a2dd22e7758651d9b6d12df09a13d51ba5bf4fc33c383a48e1775 + # via + # -r requirements.txt + # mcp +werkzeug==3.1.7 \ + --hash=sha256:4b314d81163a3e1a169b6a0be2a000a0e204e8873c5de6586f453c55688d422f \ + --hash=sha256:fb8c01fe6ab13b9b7cdb46892b99b1d66754e1d7ab8e542e865ec13f526b5351 + # via + # -r requirements.txt # flask # flask-socketio wsproto==1.3.2 \ --hash=sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584 \ --hash=sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294 - # via simple-websocket + # via + # -r requirements.txt + # simple-websocket zipp==3.23.0 \ --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 - # via importlib-metadata + # via + # -r requirements.txt + # importlib-metadata diff --git a/requirements.txt b/requirements.txt index f488270f..8eb3a261 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,204 @@ -flask==3.1.3 -flask-socketio==5.6.1 -simple-websocket==1.1.0 +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -o requirements.txt +annotated-types==0.7.0 + # via pydantic +anyio==4.13.0 + # via + # claude-agent-sdk + # httpx + # mcp + # sse-starlette + # starlette +attrs==26.1.0 + # via + # jsonschema + # referencing +bidict==0.23.1 + # via python-socketio +blinker==1.9.0 + # via + # flask + # flask-socketio +cachetools==7.0.5 + # via mlflow-tracing +certifi==2026.2.25 + # via + # httpcore + # httpx + # requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.6 + # via requests claude-agent-sdk==0.1.50 + # via coda (pyproject.toml) +click==8.3.1 + # via + # flask + # flask-socketio + # uvicorn +cryptography==46.0.6 + # via + # coda (pyproject.toml) + # google-auth + # pyjwt databricks-sdk==0.102.0 + # via + # coda (pyproject.toml) + # mlflow-tracing +flask==3.1.3 + # via + # coda (pyproject.toml) + # flask-socketio +flask-socketio==5.6.1 + # via coda (pyproject.toml) +google-auth==2.49.1 + # via databricks-sdk +googleapis-common-protos==1.73.1 + # via opentelemetry-exporter-otlp-proto-grpc +grpcio==1.80.0 + # via opentelemetry-exporter-otlp-proto-grpc +h11==0.16.0 + # via + # httpcore + # uvicorn + # wsproto +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via mcp +httpx-sse==0.4.3 + # via mcp +idna==3.11 + # via + # anyio + # httpx + # requests +importlib-metadata==8.7.1 + # via opentelemetry-api +itsdangerous==2.2.0 + # via flask +jinja2==3.1.6 + # via + # flask + # flask-socketio +jsonschema==4.26.0 + # via mcp +jsonschema-specifications==2025.9.1 + # via jsonschema +markupsafe==3.0.3 + # via + # flask + # jinja2 + # werkzeug +mcp==1.26.0 + # via claude-agent-sdk mlflow-tracing==3.10.1 + # via coda (pyproject.toml) +opentelemetry-api==1.40.0 + # via + # mlflow-tracing + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-otlp-proto-common==1.40.0 + # via opentelemetry-exporter-otlp-proto-grpc opentelemetry-exporter-otlp-proto-grpc==1.40.0 -requests @ git+https://github.com/psf/requests@v2.33.0 -cryptography==46.0.6 + # via coda (pyproject.toml) +opentelemetry-proto==1.40.0 + # via + # mlflow-tracing + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc +opentelemetry-sdk==1.40.0 + # via + # mlflow-tracing + # opentelemetry-exporter-otlp-proto-grpc +opentelemetry-semantic-conventions==0.61b0 + # via opentelemetry-sdk +packaging==26.0 + # via mlflow-tracing +protobuf==6.33.6 + # via + # databricks-sdk + # googleapis-common-protos + # mlflow-tracing + # opentelemetry-proto +pyasn1==0.6.3 + # via pyasn1-modules +pyasn1-modules==0.4.2 + # via google-auth +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # mcp + # mlflow-tracing + # pydantic-settings +pydantic-core==2.41.5 + # via pydantic +pydantic-settings==2.13.1 + # via mcp +pyjwt==2.12.1 + # via mcp +python-dotenv==1.2.2 + # via pydantic-settings +python-engineio==4.13.1 + # via python-socketio +python-multipart==0.0.22 + # via mcp +python-socketio==5.16.1 + # via flask-socketio +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +requests @ git+https://github.com/psf/requests@bc04dfd6dad4cb02cd92f5daa81eb562d280a761 + # via + # coda (pyproject.toml) + # databricks-sdk +rpds-py==0.30.0 + # via + # jsonschema + # referencing +simple-websocket==1.1.0 + # via + # coda (pyproject.toml) + # python-engineio +sse-starlette==3.3.4 + # via mcp +starlette==1.0.0 + # via + # mcp + # sse-starlette +typing-extensions==4.15.0 + # via + # anyio + # grpcio + # mcp + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # referencing + # starlette + # typing-inspection +typing-inspection==0.4.2 + # via + # mcp + # pydantic + # pydantic-settings +urllib3==2.6.3 + # via requests +uvicorn==0.42.0 + # via mcp +werkzeug==3.1.7 + # via + # flask + # flask-socketio +wsproto==1.3.2 + # via simple-websocket +zipp==3.23.0 + # via importlib-metadata From 3bb9d2e9692fc78d527f5dac92c0924cf76b481d Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Wed, 8 Apr 2026 17:54:54 -0400 Subject: [PATCH 169/382] fix: ignore GHSA-p423-j2cm-9vmq until cryptography 46.0.7 is released MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin cryptography>=46.0.6 and suppress the audit warning for the buffer overflow CVE — fix version 46.0.7 is not yet available on PyPI. --- .github/workflows/dependency-audit.yml | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index 097ffaf0..7638ea83 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -53,7 +53,8 @@ jobs: # platform-conditional deps (greenlet) missing from the lockfile. # The hashes are verified at install time, not audit time. sed '/^[[:space:]]*--hash/d' requirements.lock > /tmp/requirements.lock.nohash - pip-audit -r /tmp/requirements.lock.nohash --desc on + # GHSA-p423-j2cm-9vmq: cryptography 46.0.7 not yet released — ignore until available + pip-audit -r /tmp/requirements.lock.nohash --desc on --ignore-vuln GHSA-p423-j2cm-9vmq else echo "::warning::No requirements.lock found — auditing requirements.txt (unpinned)" pip-audit -r requirements.txt --desc on diff --git a/pyproject.toml b/pyproject.toml index 5a6b6cae..02cb9cad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "mlflow-tracing>=3.4", "opentelemetry-exporter-otlp-proto-grpc", "requests", - "cryptography", + "cryptography>=46.0.6", ] [tool.uv] From 2135d1ad7656648a73b8e4d86f0efbda5d7b580a Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Thu, 9 Apr 2026 10:04:00 -0400 Subject: [PATCH 170/382] chore(deps): bump charset-normalizer 3.4.7, claude-agent-sdk 0.1.54 (#117) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recompiled requirements.txt from pyproject.toml via uv pip compile. Both packages pass the 7-day exclude-newer supply-chain gate. - charset-normalizer 3.4.6 → 3.4.7 (released 2026-04-02) - claude-agent-sdk 0.1.50 → 0.1.54 (released 2026-04-02) Dependabot PRs #110 (pydantic-core), #111 (cryptography), #113 (claude-agent-sdk 0.1.58) propose versions blocked by the 7-day rule. PR #109 (importlib-metadata 9.0.0) also blocked by UV exclude-newer. --- requirements.lock | 274 +++++++++++++++++++++++----------------------- requirements.txt | 4 +- 2 files changed, 139 insertions(+), 139 deletions(-) diff --git a/requirements.lock b/requirements.lock index 8a00b772..1e3d8765 100644 --- a/requirements.lock +++ b/requirements.lock @@ -138,146 +138,146 @@ cffi==2.0.0 \ # via # -r requirements.txt # cryptography -charset-normalizer==3.4.6 \ - --hash=sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e \ - --hash=sha256:0c173ce3a681f309f31b87125fecec7a5d1347261ea11ebbb856fa6006b23c8c \ - --hash=sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5 \ - --hash=sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815 \ - --hash=sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f \ - --hash=sha256:150b8ce8e830eb7ccb029ec9ca36022f756986aaaa7956aad6d9ec90089338c0 \ - --hash=sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484 \ - --hash=sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407 \ - --hash=sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6 \ - --hash=sha256:1cf0a70018692f85172348fe06d3a4b63f94ecb055e13a00c644d368eb82e5b8 \ - --hash=sha256:1ed80ff870ca6de33f4d953fda4d55654b9a2b340ff39ab32fa3adbcd718f264 \ - --hash=sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815 \ - --hash=sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2 \ - --hash=sha256:259695e2ccc253feb2a016303543d691825e920917e31f894ca1a687982b1de4 \ - --hash=sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579 \ - --hash=sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f \ - --hash=sha256:2bd9d128ef93637a5d7a6af25363cf5dec3fa21cf80e68055aad627f280e8afa \ - --hash=sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95 \ - --hash=sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab \ - --hash=sha256:2f7fdd9b6e6c529d6a2501a2d36b240109e78a8ceaef5687cfcfa2bbe671d297 \ - --hash=sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a \ - --hash=sha256:31215157227939b4fb3d740cd23fe27be0439afef67b785a1eb78a3ae69cba9e \ - --hash=sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84 \ - --hash=sha256:3516bbb8d42169de9e61b8520cbeeeb716f12f4ecfe3fd30a9919aa16c806ca8 \ - --hash=sha256:3778fd7d7cd04ae8f54651f4a7a0bd6e39a0cf20f801720a4c21d80e9b7ad6b0 \ - --hash=sha256:39f5068d35621da2881271e5c3205125cc456f54e9030d3f723288c873a71bf9 \ - --hash=sha256:404a1e552cf5b675a87f0651f8b79f5f1e6fd100ee88dc612f89aa16abd4486f \ - --hash=sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1 \ - --hash=sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843 \ - --hash=sha256:4482481cb0572180b6fd976a4d5c72a30263e98564da68b86ec91f0fe35e8565 \ - --hash=sha256:461598cd852bfa5a61b09cae2b1c02e2efcd166ee5516e243d540ac24bfa68a7 \ - --hash=sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c \ - --hash=sha256:48696db7f18afb80a068821504296eb0787d9ce239b91ca15059d1d3eaacf13b \ - --hash=sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7 \ - --hash=sha256:4d1d02209e06550bdaef34af58e041ad71b88e624f5d825519da3a3308e22687 \ - --hash=sha256:4f41da960b196ea355357285ad1316a00099f22d0929fe168343b99b254729c9 \ - --hash=sha256:517ad0e93394ac532745129ceabdf2696b609ec9f87863d337140317ebce1c14 \ - --hash=sha256:51fb3c322c81d20567019778cb5a4a6f2dc1c200b886bc0d636238e364848c89 \ - --hash=sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f \ - --hash=sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0 \ - --hash=sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9 \ - --hash=sha256:54fae94be3d75f3e573c9a1b5402dc593de19377013c9a0e4285e3d402dd3a2a \ - --hash=sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389 \ - --hash=sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0 \ - --hash=sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30 \ - --hash=sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd \ - --hash=sha256:5feb91325bbceade6afab43eb3b508c63ee53579fe896c77137ded51c6b6958e \ - --hash=sha256:60c74963d8350241a79cb8feea80e54d518f72c26db618862a8f53e5023deaf9 \ - --hash=sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc \ - --hash=sha256:659a1e1b500fac8f2779dd9e1570464e012f43e580371470b45277a27baa7532 \ - --hash=sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d \ - --hash=sha256:69dd852c2f0ad631b8b60cfbe25a28c0058a894de5abb566619c205ce0550eae \ - --hash=sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2 \ - --hash=sha256:71be7e0e01753a89cf024abf7ecb6bca2c81738ead80d43004d9b5e3f1244e64 \ - --hash=sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f \ - --hash=sha256:74a2e659c7ecbc73562e2a15e05039f1e22c75b7c7618b4b574a3ea9118d1557 \ - --hash=sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e \ - --hash=sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff \ - --hash=sha256:7a6967aaf043bceabab5412ed6bd6bd26603dae84d5cb75bf8d9a74a4959d398 \ - --hash=sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db \ - --hash=sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a \ - --hash=sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43 \ - --hash=sha256:802168e03fba8bbc5ce0d866d589e4b1ca751d06edee69f7f3a19c5a9fe6b597 \ - --hash=sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c \ - --hash=sha256:82060f995ab5003a2d6e0f4ad29065b7672b6593c8c63559beefe5b443242c3e \ - --hash=sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2 \ - --hash=sha256:8761ac29b6c81574724322a554605608a9960769ea83d2c73e396f3df896ad54 \ - --hash=sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e \ - --hash=sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4 \ - --hash=sha256:8bc5f0687d796c05b1e28ab0d38a50e6309906ee09375dd3aff6a9c09dd6e8f4 \ - --hash=sha256:8bea55c4eef25b0b19a0337dc4e3f9a15b00d569c77211fa8cde38684f234fb7 \ - --hash=sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6 \ - --hash=sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5 \ - --hash=sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194 \ - --hash=sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69 \ - --hash=sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f \ - --hash=sha256:97d0235baafca5f2b09cf332cc275f021e694e8362c6bb9c96fc9a0eb74fc316 \ - --hash=sha256:9ca4c0b502ab399ef89248a2c84c54954f77a070f28e546a85e91da627d1301e \ - --hash=sha256:9cc4fc6c196d6a8b76629a70ddfcd4635a6898756e2d9cac5565cf0654605d73 \ - --hash=sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8 \ - --hash=sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923 \ - --hash=sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88 \ - --hash=sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f \ - --hash=sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21 \ - --hash=sha256:a9e68c9d88823b274cf1e72f28cb5dc89c990edf430b0bfd3e2fb0785bfeabf4 \ - --hash=sha256:aa9cccf4a44b9b62d8ba8b4dd06c649ba683e4bf04eea606d2e94cfc2d6ff4d6 \ - --hash=sha256:ab30e5e3e706e3063bc6de96b118688cb10396b70bb9864a430f67df98c61ecc \ - --hash=sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2 \ - --hash=sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866 \ - --hash=sha256:b35b200d6a71b9839a46b9b7fff66b6638bb52fc9658aa58796b0326595d3021 \ - --hash=sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2 \ - --hash=sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d \ - --hash=sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8 \ - --hash=sha256:bc72863f4d9aba2e8fd9085e63548a324ba706d2ea2c83b260da08a59b9482de \ - --hash=sha256:bf625105bb9eef28a56a943fec8c8a98aeb80e7d7db99bd3c388137e6eb2d237 \ - --hash=sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4 \ - --hash=sha256:c45a03a4c69820a399f1dda9e1d8fbf3562eda46e7720458180302021b08f778 \ - --hash=sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb \ - --hash=sha256:c907cdc8109f6c619e6254212e794d6548373cc40e1ec75e6e3823d9135d29cc \ - --hash=sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602 \ - --hash=sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4 \ - --hash=sha256:d08ec48f0a1c48d75d0356cea971921848fb620fdeba805b28f937e90691209f \ - --hash=sha256:d1a2ee9c1499fc8f86f4521f27a973c914b211ffa87322f4ee33bb35392da2c5 \ - --hash=sha256:d5f5d1e9def3405f60e3ca8232d56f35c98fb7bf581efcc60051ebf53cb8b611 \ - --hash=sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8 \ - --hash=sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf \ - --hash=sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d \ - --hash=sha256:dad6e0f2e481fffdcf776d10ebee25e0ef89f16d691f1e5dee4b586375fdc64b \ - --hash=sha256:dda86aba335c902b6149a02a55b38e96287157e609200811837678214ba2b1db \ - --hash=sha256:df01808ee470038c3f8dc4f48620df7225c49c2d6639e38f96e6d6ac6e6f7b0e \ - --hash=sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077 \ - --hash=sha256:e25369dc110d58ddf29b949377a93e0716d72a24f62bad72b2b39f155949c1fd \ - --hash=sha256:e3c701e954abf6fc03a49f7c579cc80c2c6cc52525340ca3186c41d3f33482ef \ - --hash=sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e \ - --hash=sha256:e68c14b04827dd76dcbd1aeea9e604e3e4b78322d8faf2f8132c7138efa340a8 \ - --hash=sha256:e8aeb10fcbe92767f0fa69ad5a72deca50d0dca07fbde97848997d778a50c9fe \ - --hash=sha256:e985a16ff513596f217cee86c21371b8cd011c0f6f056d0920aa2d926c544058 \ - --hash=sha256:ecbbd45615a6885fe3240eb9db73b9e62518b611850fdf8ab08bd56de7ad2b17 \ - --hash=sha256:ee4ec14bc1680d6b0afab9aea2ef27e26d2024f18b24a2d7155a52b60da7e833 \ - --hash=sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421 \ - --hash=sha256:f0cdaecd4c953bfae0b6bb64910aaaca5a424ad9c72d85cb88417bb9814f7550 \ - --hash=sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff \ - --hash=sha256:f50498891691e0864dc3da965f340fada0771f6142a378083dc4608f4ea513e2 \ - --hash=sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc \ - --hash=sha256:f61aa92e4aad0be58eb6eb4e0c21acf32cf8065f4b2cae5665da756c4ceef982 \ - --hash=sha256:f6e4333fb15c83f7d1482a76d45a0818897b3d33f00efd215528ff7c51b8e35d \ - --hash=sha256:f820f24b09e3e779fe84c3c456cb4108a7aa639b0d1f02c28046e11bfcd088ed \ - --hash=sha256:f98059e4fcd3e3e4e2d632b7cf81c2faae96c43c60b569e9c621468082f1d104 \ - --hash=sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659 +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 # via # -r requirements.txt # requests -claude-agent-sdk==0.1.50 \ - --hash=sha256:2e44caf3e5bce56e26a18158acf3e1c2c2784cf8fa15e425afe92816c987eb1a \ - --hash=sha256:44e75b9d076bd6030742729f99eb38777b80f052b22338d0a028d8190fc59e52 \ - --hash=sha256:493d8cc43f4166291606749cf47b03e822f03b7f371cc77af697564017ccf579 \ - --hash=sha256:7363d431dc6efd83fa658a045e14fa4357440352b548002bfb9096d8f04d143c \ - --hash=sha256:858b1822451209b2c3ad8df27458168d29ac19fd628680853f7707ea017fea73 \ - --hash=sha256:e15157792857ecb55274a71f08981efcfda2e169bee7894cbdc245d05ac43203 +claude-agent-sdk==0.1.54 \ + --hash=sha256:4b275e1cd8be7cc112013fa4226559f940fea37577d0bea26a0a980d46e62dbb \ + --hash=sha256:6c54fed15ef7801cb4efe76eaa6a79ae83ba1faaf002a0da78788595b05b260c \ + --hash=sha256:931152076712dcc8980cdc046eed6d9ef85efe8ed7bff2a6a7a29ef70dc5d6d7 \ + --hash=sha256:d6858685e745eddbd53330668370109ea566bc250a6354bd28e1e46790439af8 \ + --hash=sha256:e51d901ac3c4a4f81966383aef120f4bd96f385edd56b7abc6cdd6700d8c8d02 \ + --hash=sha256:e7519a30f351581eae260e2087aec06e0cd231aa82088d6ddeeb5692614773b1 # via -r requirements.txt click==8.3.1 \ --hash=sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a \ diff --git a/requirements.txt b/requirements.txt index 8eb3a261..b7f405c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,9 +28,9 @@ certifi==2026.2.25 # requests cffi==2.0.0 # via cryptography -charset-normalizer==3.4.6 +charset-normalizer==3.4.7 # via requests -claude-agent-sdk==0.1.50 +claude-agent-sdk==0.1.54 # via coda (pyproject.toml) click==8.3.1 # via From 291f8fb0801f5a03686a4080617dfa733a78b6c7 Mon Sep 17 00:00:00 2001 From: Sathish Gangichetty Date: Tue, 14 Apr 2026 14:33:03 -0400 Subject: [PATCH 171/382] feat: session management, clipboard, rendering fixes, dep cleanup (v0.17.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #118, #120, #121, #122 - Session creation prompt: ask users to reuse existing sessions before creating new - MAX_CONCURRENT_SESSIONS backend cap (env var, default 5) with TOCTOU-safe check - Session count label in tab bar with updates on all create/close/exit paths - xterm.js ClipboardAddon for OSC 52 (copy-paste inside Claude Code) - Write batching with requestAnimationFrame to prevent escape sequence fragmentation - Alternate screen exit detection (auto-clear after Claude Code no-flicker/vim) - SIGWINCH-based reattach (force redraw by toggling terminal size) - 429 error message with hint to increase MAX_CONCURRENT_SESSIONS - Replaced mlflow-tracing with mlflow-skinny 3.10.1 - PTY read chunk 4096→65536 bytes - Fixed repo name in deployment docs - Version bump to 0.17.0 --- README.md | 2 +- app.py | 17 ++- app.yaml | 2 + docs/deployment.md | 8 +- pyproject.toml | 5 +- requirements.txt | 70 ++++++----- static/index.html | 186 +++++++++++++++++++++++++--- static/lib/addon-clipboard.js | 2 + tests/test_clipboard_addon.py | 76 ++++++++++++ tests/test_session_limit.py | 223 ++++++++++++++++++++++++++++++++++ 10 files changed, 533 insertions(+), 58 deletions(-) create mode 100644 static/lib/addon-clipboard.js create mode 100644 tests/test_clipboard_addon.py create mode 100644 tests/test_session_limit.py diff --git a/README.md b/README.md index 87004caa..50ef0225 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ Production uses `workers=1` (PTY state is process-local), `threads=16` (concurre 📁 Project Structure ``` -coding-agents-in-databricks/ +coding-agents-databricks-apps/ ├── app.py # Flask backend + PTY management + setup orchestration ├── app_state.py # Shared app state (setup progress, session registry) ├── app.yaml.template # Databricks Apps deployment config template diff --git a/app.py b/app.py index 514c4bca..71ac4f45 100644 --- a/app.py +++ b/app.py @@ -43,6 +43,7 @@ SESSION_TIMEOUT_SECONDS = 86400 # No poll for 24 hours = dead session CLEANUP_INTERVAL_SECONDS = 900 # Check for stale sessions every 15 min GRACEFUL_SHUTDOWN_WAIT = 3 # Seconds to wait after SIGHUP before SIGKILL +MAX_CONCURRENT_SESSIONS = int(os.environ.get("MAX_CONCURRENT_SESSIONS", "5")) # Logging setup logging.basicConfig(level=logging.INFO) @@ -604,7 +605,7 @@ def read_pty_output(session_id, fd): try: readable, _, errors = select.select([fd], [], [fd], 0.05) if readable or errors: - output = os.read(fd, 4096) + output = os.read(fd, 65536) if not output: # EOF — process exited break @@ -956,6 +957,11 @@ def configure_pat(): @app.route("/api/session", methods=["POST"]) def create_session(): """Create a new terminal session.""" + # Quick reject before forking a PTY (approximate — authoritative check below) + with sessions_lock: + if len(sessions) >= MAX_CONCURRENT_SESSIONS: + return jsonify({"error": f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached. Close an existing session first."}), 429 + data = request.get_json(silent=True) or {} label = data.get("label", "") try: @@ -997,6 +1003,15 @@ def create_session(): session_id = str(uuid.uuid4()) with sessions_lock: + # Authoritative check under the same lock as insertion — prevents + # TOCTOU race where two concurrent requests both pass the early check. + if len(sessions) >= MAX_CONCURRENT_SESSIONS: + os.close(master_fd) + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + return jsonify({"error": f"Maximum {MAX_CONCURRENT_SESSIONS} concurrent sessions reached. Close an existing session first."}), 429 sessions[session_id] = { "master_fd": master_fd, "pid": pid, diff --git a/app.yaml b/app.yaml index e6bb8cde..5596e08e 100644 --- a/app.yaml +++ b/app.yaml @@ -12,3 +12,5 @@ env: value: databricks-gpt-5-3-codex - name: CLAUDE_CODE_DISABLE_AUTO_MEMORY value: 0 + - name: MAX_CONCURRENT_SESSIONS + value: "5" diff --git a/docs/deployment.md b/docs/deployment.md index 09959da1..36267c6f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -11,7 +11,7 @@ The simplest way — no CLI, no cloning, everything stays in the Databricks UI. 1. Go to **Databricks → Apps → Create App** 2. Choose **Custom App** and connect this Git repo: ``` - https://github.com/datasciencemonkey/coding-agents-in-databricks.git + https://github.com/datasciencemonkey/coding-agents-databricks-apps.git ``` 3. Click **Deploy** 4. Open the app — on first terminal session, paste a short-lived PAT when prompted @@ -30,8 +30,8 @@ If you prefer working from the terminal or need more control: ```bash databricks repos create \ - --url https://github.com/datasciencemonkey/coding-agents-in-databricks.git \ - --path /Workspace/Users//apps/coding-agents-in-databricks + --url https://github.com/datasciencemonkey/coding-agents-databricks-apps.git \ + --path /Workspace/Users//apps/coding-agents-databricks-apps ``` ### 2. Configure `app.yaml` @@ -56,7 +56,7 @@ No secrets or resources to configure. On first terminal session, paste a short-l ```bash databricks apps deploy \ - --source-code-path /Workspace/Users//apps/coding-agents-in-databricks + --source-code-path /Workspace/Users//apps/coding-agents-databricks-apps ``` > **Tip:** To update later, just `git pull` in the workspace repo and re-deploy. diff --git a/pyproject.toml b/pyproject.toml index 02cb9cad..774238c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coda" -version = "0.16.7" +version = "0.17.0" description = "CoDA - Coding Agents on Databricks Apps" requires-python = ">=3.10" dependencies = [ @@ -9,8 +9,7 @@ dependencies = [ "simple-websocket>=1.0", "claude-agent-sdk", "databricks-sdk>=0.20.0", - "mlflow-tracing>=3.4", - "opentelemetry-exporter-otlp-proto-grpc", + "mlflow-skinny==3.10.1", "requests", "cryptography>=46.0.6", ] diff --git a/requirements.txt b/requirements.txt index b7f405c5..46f445a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml -o requirements.txt +annotated-doc==0.0.4 + # via fastapi annotated-types==0.7.0 # via pydantic anyio==4.13.0 @@ -20,7 +22,7 @@ blinker==1.9.0 # flask # flask-socketio cachetools==7.0.5 - # via mlflow-tracing + # via mlflow-skinny certifi==2026.2.25 # via # httpcore @@ -36,7 +38,10 @@ click==8.3.1 # via # flask # flask-socketio + # mlflow-skinny # uvicorn +cloudpickle==3.1.2 + # via mlflow-skinny cryptography==46.0.6 # via # coda (pyproject.toml) @@ -45,19 +50,21 @@ cryptography==46.0.6 databricks-sdk==0.102.0 # via # coda (pyproject.toml) - # mlflow-tracing + # mlflow-skinny +fastapi==0.135.3 + # via mlflow-skinny flask==3.1.3 # via # coda (pyproject.toml) # flask-socketio flask-socketio==5.6.1 # via coda (pyproject.toml) +gitdb==4.0.12 + # via gitpython +gitpython==3.1.46 + # via mlflow-skinny google-auth==2.49.1 # via databricks-sdk -googleapis-common-protos==1.73.1 - # via opentelemetry-exporter-otlp-proto-grpc -grpcio==1.80.0 - # via opentelemetry-exporter-otlp-proto-grpc h11==0.16.0 # via # httpcore @@ -75,7 +82,9 @@ idna==3.11 # httpx # requests importlib-metadata==8.7.1 - # via opentelemetry-api + # via + # mlflow-skinny + # opentelemetry-api itsdangerous==2.2.0 # via flask jinja2==3.1.6 @@ -93,36 +102,25 @@ markupsafe==3.0.3 # werkzeug mcp==1.26.0 # via claude-agent-sdk -mlflow-tracing==3.10.1 +mlflow-skinny==3.10.1 # via coda (pyproject.toml) opentelemetry-api==1.40.0 # via - # mlflow-tracing - # opentelemetry-exporter-otlp-proto-grpc + # mlflow-skinny # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp-proto-common==1.40.0 - # via opentelemetry-exporter-otlp-proto-grpc -opentelemetry-exporter-otlp-proto-grpc==1.40.0 - # via coda (pyproject.toml) opentelemetry-proto==1.40.0 - # via - # mlflow-tracing - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc + # via mlflow-skinny opentelemetry-sdk==1.40.0 - # via - # mlflow-tracing - # opentelemetry-exporter-otlp-proto-grpc + # via mlflow-skinny opentelemetry-semantic-conventions==0.61b0 # via opentelemetry-sdk packaging==26.0 - # via mlflow-tracing + # via mlflow-skinny protobuf==6.33.6 # via # databricks-sdk - # googleapis-common-protos - # mlflow-tracing + # mlflow-skinny # opentelemetry-proto pyasn1==0.6.3 # via pyasn1-modules @@ -132,8 +130,9 @@ pycparser==3.0 # via cffi pydantic==2.12.5 # via + # fastapi # mcp - # mlflow-tracing + # mlflow-skinny # pydantic-settings pydantic-core==2.41.5 # via pydantic @@ -142,13 +141,17 @@ pydantic-settings==2.13.1 pyjwt==2.12.1 # via mcp python-dotenv==1.2.2 - # via pydantic-settings + # via + # mlflow-skinny + # pydantic-settings python-engineio==4.13.1 # via python-socketio python-multipart==0.0.22 # via mcp python-socketio==5.16.1 # via flask-socketio +pyyaml==6.0.3 + # via mlflow-skinny referencing==0.37.0 # via # jsonschema @@ -157,6 +160,7 @@ requests @ git+https://github.com/psf/requests@bc04dfd6dad4cb02cd92f5daa81eb562d # via # coda (pyproject.toml) # databricks-sdk + # mlflow-skinny rpds-py==0.30.0 # via # jsonschema @@ -165,19 +169,24 @@ simple-websocket==1.1.0 # via # coda (pyproject.toml) # python-engineio +smmap==5.0.3 + # via gitdb +sqlparse==0.5.5 + # via mlflow-skinny sse-starlette==3.3.4 # via mcp starlette==1.0.0 # via + # fastapi # mcp # sse-starlette typing-extensions==4.15.0 # via # anyio - # grpcio + # fastapi # mcp + # mlflow-skinny # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-sdk # opentelemetry-semantic-conventions # pydantic @@ -187,13 +196,16 @@ typing-extensions==4.15.0 # typing-inspection typing-inspection==0.4.2 # via + # fastapi # mcp # pydantic # pydantic-settings urllib3==2.6.3 # via requests uvicorn==0.42.0 - # via mcp + # via + # mcp + # mlflow-skinny werkzeug==3.1.7 # via # flask diff --git a/static/index.html b/static/index.html index bb7eecfd..d27965e8 100644 --- a/static/index.html +++ b/static/index.html @@ -142,6 +142,11 @@ border-color: rgba(100,150,255,0.25); box-shadow: 0 0 8px rgba(100,150,255,0.1); } + #session-count-label { + margin-left: auto; padding: 0 12px; + font-size: 11px; color: rgba(255,255,255,0.4); + white-space: nowrap; flex-shrink: 0; + } #toolbar .font-size-row { display: flex; align-items: center; gap: 4px; } @@ -375,6 +380,7 @@

General

+
@@ -383,6 +389,7 @@

General

+