From 26a61e1c442e16d97cbca8eabb9492b51330aaf3 Mon Sep 17 00:00:00 2001 From: R4ph-t <1368445+R4ph-t@users.noreply.github.com> Date: Fri, 1 May 2026 07:06:00 +0000 Subject: [PATCH] Sync skills from render-oss/skills --- .../render-background-workers/evals.json | 57 ---- .../skills/render-blueprints/evals.json | 57 ---- plugins/render/skills/render-cli/evals.json | 76 ----- .../render/skills/render-cron-jobs/evals.json | 83 ------ plugins/render/skills/render-debug/evals.json | 259 ------------------ .../render/skills/render-deploy/evals.json | 91 ------ plugins/render/skills/render-disks/evals.json | 52 ---- .../render/skills/render-docker/evals.json | 57 ---- .../render/skills/render-domains/evals.json | 52 ---- .../render/skills/render-env-vars/evals.json | 83 ------ .../render/skills/render-keyvalue/evals.json | 51 ---- plugins/render/skills/render-mcp/evals.json | 53 ---- .../render-migrate-from-heroku/evals.json | 57 ---- .../render/skills/render-monitor/evals.json | 52 ---- .../skills/render-networking/evals.json | 55 ---- .../render/skills/render-postgres/evals.json | 128 --------- .../skills/render-private-services/evals.json | 54 ---- .../render/skills/render-scaling/evals.json | 102 ------- .../skills/render-static-sites/evals.json | 46 ---- .../skills/render-web-services/evals.json | 56 ---- .../render/skills/render-workflows/evals.json | 54 ---- 21 files changed, 1575 deletions(-) delete mode 100644 plugins/render/skills/render-background-workers/evals.json delete mode 100644 plugins/render/skills/render-blueprints/evals.json delete mode 100644 plugins/render/skills/render-cli/evals.json delete mode 100644 plugins/render/skills/render-cron-jobs/evals.json delete mode 100644 plugins/render/skills/render-debug/evals.json delete mode 100644 plugins/render/skills/render-deploy/evals.json delete mode 100644 plugins/render/skills/render-disks/evals.json delete mode 100644 plugins/render/skills/render-docker/evals.json delete mode 100644 plugins/render/skills/render-domains/evals.json delete mode 100644 plugins/render/skills/render-env-vars/evals.json delete mode 100644 plugins/render/skills/render-keyvalue/evals.json delete mode 100644 plugins/render/skills/render-mcp/evals.json delete mode 100644 plugins/render/skills/render-migrate-from-heroku/evals.json delete mode 100644 plugins/render/skills/render-monitor/evals.json delete mode 100644 plugins/render/skills/render-networking/evals.json delete mode 100644 plugins/render/skills/render-postgres/evals.json delete mode 100644 plugins/render/skills/render-private-services/evals.json delete mode 100644 plugins/render/skills/render-scaling/evals.json delete mode 100644 plugins/render/skills/render-static-sites/evals.json delete mode 100644 plugins/render/skills/render-web-services/evals.json delete mode 100644 plugins/render/skills/render-workflows/evals.json diff --git a/plugins/render/skills/render-background-workers/evals.json b/plugins/render/skills/render-background-workers/evals.json deleted file mode 100644 index 3104555..0000000 --- a/plugins/render/skills/render-background-workers/evals.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "skill_name": "render-background-workers", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I need to set up a BullMQ worker in Node.js on Render to process async jobs. I want it connected to a Redis-compatible queue. Can you give me a complete render.yaml blueprint and explain the Key Value eviction policy I should use?", - "expected_output": "The agent provides a complete render.yaml with a Key Value service (ipAllowList, noeviction policy note) and a worker service (type: worker, runtime: node, buildCommand: npm ci, startCommand: node worker.js) with REDIS_URL wired via fromService. It explains that maxmemory-policy must be set to noeviction to prevent queue keys from being evicted.", - "files": [], - "integrations": {}, - "assertions": [ - "The response includes a render.yaml with type: keyvalue and type: worker services", - "The worker service uses runtime: node and startCommand referencing node worker.js or equivalent", - "REDIS_URL is wired using fromService with type: keyvalue and property: connectionString", - "The response explicitly states the Key Value maxmemory policy must be set to noeviction", - "The Key Value service includes an ipAllowList field", - "The response explains that allkeys-lru or similar eviction policies are for caches and will drop jobs" - ] - }, - { - "id": 2, - "turns": [ - "I want to run background jobs on Render. Should I use a background worker or a cron job?", - "The jobs are triggered by user actions and need to be processed as soon as possible. There could be bursts of hundreds of jobs at once." - ], - "expected_output": "The agent first asks clarifying questions about the job trigger pattern (scheduled vs event-driven) and volume. After the user clarifies that jobs are user-triggered and bursty, the agent recommends a Background Worker for always-on queue consumption, or a Workflow for high-volume bursty scenarios, explaining the tradeoffs. It explains cron jobs are for scheduled periodic tasks that exit, not queue consumers.", - "files": [], - "integrations": {}, - "assertions": [ - "In the first turn, the agent asks at least one clarifying question about the job trigger pattern or volume", - "After the user clarifies bursty user-triggered jobs, the agent recommends a Background Worker or Workflow", - "The agent explains that Cron Jobs are for scheduled periodic tasks and have a 12-hour max run limit", - "The agent mentions that Workflows scale per run and have no idle instance cost, making them suitable for bursty workloads", - "The agent distinguishes between always-on queue consumers (Background Worker) and scheduled tasks (Cron Job)" - ] - }, - { - "id": 3, - "prompt": "My Celery workers on Render are losing jobs during deploys. How do I implement graceful shutdown? I need jobs that can take up to 2 minutes to finish. Show me the signal handler code and the render.yaml config.", - "expected_output": "The agent explains Render's SIGTERM -> wait -> SIGKILL sequence, provides a Python SIGTERM signal handler using signal.signal, recommends setting maxShutdownDelaySeconds to at least 120 (2 minutes) plus buffer, and shows the render.yaml worker config with maxShutdownDelaySeconds set appropriately. It also mentions Celery's worker_shutting_down lifecycle signal and the importance of idempotent tasks.", - "files": [], - "integrations": {}, - "assertions": [ - "The response explains that Render sends SIGTERM first, waits maxShutdownDelaySeconds, then sends SIGKILL", - "The response recommends setting maxShutdownDelaySeconds to at least 120 seconds (2 minutes) to cover the longest job duration", - "The response includes a Python signal handler using signal.signal(signal.SIGTERM, ...) or references Celery's worker_shutting_down signal", - "The render.yaml snippet includes maxShutdownDelaySeconds on the worker service set to 120 or higher", - "The response advises stopping the consumer loop (stop dequeuing) as the first step on SIGTERM", - "The response mentions job idempotency or checkpointing to handle retries safely" - ] - } - ] -} diff --git a/plugins/render/skills/render-blueprints/evals.json b/plugins/render/skills/render-blueprints/evals.json deleted file mode 100644 index 0d35297..0000000 --- a/plugins/render/skills/render-blueprints/evals.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "skill_name": "render-blueprints", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "Write a render.yaml for a Node.js web API, a background worker, a Redis cache, and a PostgreSQL database. Wire DATABASE_URL and REDIS_URL into both the API and worker using fromDatabase/fromService. Generate a JWT_SECRET automatically. Add a STRIPE_KEY that won't sync from the repo.", - "expected_output": "A valid render.yaml with a web service (node), a worker service (node), a keyvalue service with ipAllowList, and a database. Both web and worker have DATABASE_URL wired via fromDatabase connectionString, REDIS_URL via fromService keyvalue connectionString, JWT_SECRET via generateValue, and STRIPE_KEY via sync: false.", - "files": [], - "integrations": {}, - "assertions": [ - "The YAML includes a databases entry and a keyvalue service with ipAllowList defined", - "Both the web and worker services have DATABASE_URL using fromDatabase with property: connectionString pointing to the database name", - "Both the web and worker services have REDIS_URL using fromService with type: keyvalue and property: connectionString", - "JWT_SECRET uses generateValue: true on at least one service", - "STRIPE_KEY uses sync: false on at least one service", - "No deprecated fields are used (no 'env:', 'redis' type, 'autoDeploy', or 'pullRequestPreviewsEnabled')" - ] - }, - { - "id": 2, - "turns": [ - "I want to set up preview environments in my render.yaml for PRs. Can you help?", - "We want previews created automatically for every PR, deleted after 5 days. The billing worker should NOT run in previews. The web service should use the 'starter' plan for previews. We're on the starter plan for production too." - ], - "expected_output": "A render.yaml snippet or complete file with top-level previews.generation set to 'automatic', expireAfterDays: 5, the billing worker service having a service-level previews.generation: off override, and the web service having previewPlan: starter. The agent asks a clarifying question in the first turn about the desired preview behavior before producing the config.", - "files": [], - "integrations": {}, - "assertions": [ - "In the first turn, the agent asks at least one clarifying question about preview configuration details (generation mode, expiry, service exclusions, or plan)", - "After the user's reply, the agent sets top-level previews.generation to 'automatic'", - "The top-level previews block includes expireAfterDays: 5", - "The worker service includes a service-level previews block with generation: off", - "The web service includes previewPlan: starter", - "The agent does not mix flexible and non-flexible plan families between plan and previewPlan" - ] - }, - { - "id": 3, - "prompt": "I have this render.yaml and I'm getting errors. Fix any issues you see:\n\nservices:\n - type: web\n name: api\n env: node\n autoDeploy: true\n envVars:\n - key: DB_URL\n fromDatabase:\n name: mydb\n property: connectionString\n - key: CACHE\n fromService:\n name: cache\n type: redis\n property: connectionString\ndatabases:\n - name: mydb\n region: oregon\n readReplicas: []", - "expected_output": "The agent identifies and fixes: 'env' replaced with 'runtime', 'autoDeploy' replaced with 'autoDeployTrigger', 'type: redis' replaced with 'type: keyvalue' in fromService, and warns that readReplicas: [] will destroy all replicas. The agent produces a corrected render.yaml.", - "files": [], - "integrations": {}, - "assertions": [ - "The corrected YAML replaces 'env: node' with 'runtime: node'", - "The corrected YAML replaces 'autoDeploy: true' with 'autoDeployTrigger: commit' or another valid value", - "The corrected YAML changes the fromService type from 'redis' to 'keyvalue'", - "The agent warns that 'readReplicas: []' is destructive and will remove all read replicas", - "The agent recommends running 'render blueprints validate' or validating against the JSON Schema URL" - ] - } - ] -} diff --git a/plugins/render/skills/render-cli/evals.json b/plugins/render/skills/render-cli/evals.json deleted file mode 100644 index a88673f..0000000 --- a/plugins/render/skills/render-cli/evals.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "skill_name": "render-cli", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I need a GitHub Actions workflow that deploys my Render service srv-abc123 on every push to main. It should fail the job if the deploy fails, and use my API key stored as a secret called RENDER_API_KEY.", - "expected_output": "A complete GitHub Actions YAML workflow that installs the Render CLI, authenticates via RENDER_API_KEY, runs `render deploys create srv-abc123 --wait --confirm -o json`, and exits non-zero on failure.", - "files": [], - "integrations": {}, - "assertions": [ - "The workflow triggers on push to the main branch", - "The workflow installs the Render CLI using curl or a direct download from GitHub releases", - "The RENDER_API_KEY secret is passed as an environment variable to the deploy step", - "The deploy command includes `--wait`, `--confirm`, and `-o json` flags", - "The service ID srv-abc123 is used in the `render deploys create` command", - "The workflow is valid YAML with correct GitHub Actions syntax (on, jobs, steps structure)" - ] - }, - { - "id": 2, - "turns": [ - "How do I connect to my Render Postgres database from the terminal?", - "I want to run a one-off query to count rows in the users table and get the result as JSON. My database ID is db-prod99." - ], - "expected_output": "After the user clarifies they want a one-off JSON query, the agent provides the exact `render psql` command: `render psql db-prod99 -c \"SELECT count(*) FROM users;\" -o json`, explaining the flags used.", - "files": [], - "integrations": { - "render": { - "state": { - "postgres": [ - { - "id": "db-prod99", - "name": "prod-database", - "status": "available", - "databaseName": "myapp", - "databaseUser": "myapp_user" - } - ] - }, - "overrides": { - "tool:get_postgres": { - "responseFromState": "postgres[id]" - } - } - } - }, - "assertions": [ - "In the first turn, the agent explains both interactive (`render psql db-xxx`) and non-interactive options, or asks a clarifying question about the use case", - "After the user specifies a one-off JSON query, the agent provides a command using `render psql db-prod99`", - "The command includes `-c \"SELECT count(*) FROM users;\"`", - "The command includes `-o json` for JSON output", - "The agent does not suggest using `--csv` or plain text flags for this JSON use case" - ] - }, - { - "id": 3, - "prompt": "My CI pipeline keeps hanging waiting for user input when I run `render services`. I'm also getting auth errors. How do I fix both issues so the CLI works in a non-interactive CI environment?", - "expected_output": "The agent explains: (1) set RENDER_API_KEY env var instead of using `render login` for CI auth, (2) add `--confirm` flag and `-o json` or set RENDER_OUTPUT=json to suppress interactive prompts, and optionally run `render workspace set` to configure the workspace.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent instructs the user to set the RENDER_API_KEY environment variable for non-interactive authentication", - "The agent explains that `render login` is for interactive use and should not be used in CI", - "The agent recommends adding `--confirm` to skip confirmation prompts", - "The agent recommends using `-o json` or `-o text` (or setting RENDER_OUTPUT) instead of the default interactive output mode", - "The agent mentions that `render workspace set` may be needed to configure the active workspace", - "The agent does not suggest using `--output interactive` in CI" - ] - } - ] -} diff --git a/plugins/render/skills/render-cron-jobs/evals.json b/plugins/render/skills/render-cron-jobs/evals.json deleted file mode 100644 index 08ad7fd..0000000 --- a/plugins/render/skills/render-cron-jobs/evals.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "skill_name": "render-cron-jobs", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I need to set up a cron job on Render that runs a Python database cleanup script every weekday at 9 PM New York time. Can you write the render.yaml Blueprint for it? The script is at scripts/cleanup.py and needs a DATABASE_URL env var from a database called prod-db.", - "expected_output": "The agent should provide a valid render.yaml Blueprint with type: cron, convert 9 PM Eastern (EST is UTC-5, EDT is UTC-4) to UTC (either 2:00 AM UTC as a note about DST, or recommend 02:00 UTC), use the weekday expression '0 2 * * 1-5', include buildCommand, startCommand, and envVars wired to prod-db. The agent should warn about the UTC conversion and DST ambiguity.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent converts 9 PM New York time to UTC and explains the UTC offset (EST = UTC-5, EDT = UTC-4)", - "The generated YAML uses 'type: cron' and a quoted schedule field", - "The schedule expression uses '1-5' for weekdays in the day-of-week field", - "The YAML includes 'startCommand: python scripts/cleanup.py' and a buildCommand installing dependencies", - "The envVars section references prod-db using 'fromDatabase' with property: connectionString", - "The agent warns about DST ambiguity or recommends picking a UTC time that accounts for both EST and EDT" - ] - }, - { - "id": 2, - "turns": [ - "I want to migrate my Heroku Scheduler jobs to Render. I have a few tasks running on different schedules.", - "I have three tasks: 'rake db:cleanup' runs every 10 minutes, 'python scripts/report.py' runs every day, and 'node scripts/sync.js' runs every hour. All are in the same repo." - ], - "expected_output": "After the user provides task details, the agent should produce a render.yaml with three separate cron job services, mapping Heroku's preset schedules to proper cron expressions (*/10 * * * *, 0 0 * * *, 0 * * * *), with appropriate buildCommands and startCommands for each runtime (Ruby, Python, Node.js).", - "files": [], - "integrations": {}, - "assertions": [ - "The agent asks a clarifying question after the first message to gather task details before proceeding", - "After the user provides the three tasks, the agent maps each Heroku preset to the correct cron expression: */10 * * * *, 0 * * * *, and 0 0 * * *", - "The YAML includes three separate 'type: cron' service entries, one per task", - "The Ruby task uses 'bundle exec rake db:cleanup' as startCommand and 'bundle install' as buildCommand", - "The Node.js task uses 'node scripts/sync.js' as startCommand and 'npm ci' as buildCommand", - "The agent notes key differences between Heroku Scheduler and Render cron jobs, such as the single-run guarantee or the 12-hour limit" - ] - }, - { - "id": 3, - "prompt": "My Render cron job is supposed to run every hour but it hasn't fired in 6 hours. I also want to manually trigger it now. What could be wrong and how do I trigger it? Also, can a cron job run two instances at the same time if it's slow?", - "expected_output": "The agent should troubleshoot possible causes (UTC timezone misconfiguration in the expression, job still running from a previous invocation due to single-run guarantee, service paused/suspended), explain how to manually trigger a run via Dashboard 'Trigger Run' button (which cancels any active run and starts a new one), and clarify that Render enforces a single-run guarantee so two instances cannot overlap.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "svc-cron-abc123", - "name": "hourly-job", - "type": "cron", - "schedule": "0 * * * *", - "suspended": "not_suspended", - "status": "running" - } - ] - }, - "overrides": { - "GET /services/{serviceId}": { - "responseFromState": "services[serviceId]" - }, - "tool:get_service": { - "responseFromState": "services[serviceId]" - }, - "tool:list_services": { - "responseFromState": "services" - } - } - } - }, - "assertions": [ - "The agent identifies that cron expressions are evaluated in UTC and asks the user to verify their expression is in UTC, not local time", - "The agent explains the single-run guarantee: at most one active run at a time, so a slow previous run may be blocking new invocations", - "The agent explains that manually triggering a run via 'Trigger Run' in the Dashboard will cancel any currently active run and start a new one", - "The agent confirms that two instances of the same cron job cannot run simultaneously on Render", - "The agent suggests checking service logs in the Dashboard to see if a run is currently in progress or if there are error exit codes" - ] - } - ] -} diff --git a/plugins/render/skills/render-debug/evals.json b/plugins/render/skills/render-debug/evals.json deleted file mode 100644 index 5db41b8..0000000 --- a/plugins/render/skills/render-debug/evals.json +++ /dev/null @@ -1,259 +0,0 @@ -{ - "skill_name": "render-debug", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "My Render web service 'api-prod' (ID: srv-abc123) keeps failing to deploy. The logs show: `ReferenceError: DATABASE_URL is not defined` and `Process exited with code 1`. Can you debug this and fix it?", - "expected_output": "The agent identifies a MISSING_ENV_VAR error, retrieves logs to confirm the missing DATABASE_URL, and applies a fix by calling update_environment_variables or advising to update render.yaml with a fromDatabase reference. It then verifies the fix by checking the latest deploy status and error logs.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "srv-abc123", - "name": "api-prod", - "status": "build_failed", - "type": "web_service", - "envVars": [] - } - ], - "deploys": [ - { - "id": "dep-001", - "serviceId": "srv-abc123", - "status": "build_failed", - "finishedAt": "2024-01-24T10:30:00Z" - } - ], - "logs": [ - { - "serviceId": "srv-abc123", - "level": "error", - "message": "ReferenceError: DATABASE_URL is not defined\n at connectDatabase (/app/database.js:15:23)\n at startup (/app/server.js:10:5)", - "timestamp": "2024-01-24T10:29:55Z" - }, - { - "serviceId": "srv-abc123", - "level": "error", - "message": "Process exited with code 1", - "timestamp": "2024-01-24T10:29:56Z" - } - ] - }, - "overrides": { - "tool:list_services": { - "responseFromState": "services" - }, - "tool:get_service": { - "responseFromState": "services[serviceId]" - }, - "tool:list_deploys": { - "responseFromState": "deploys[serviceId]" - }, - "tool:list_logs": { - "responseFromState": "logs" - }, - "tool:update_environment_variables": { - "response": { - "success": true, - "message": "Environment variables updated. Redeploy triggered." - } - } - } - } - }, - "assertions": [ - "The agent calls list_logs or get_service to retrieve error details for srv-abc123", - "The agent correctly identifies the error as MISSING_ENV_VAR (DATABASE_URL not defined)", - "The agent calls update_environment_variables with key DATABASE_URL or advises adding it to render.yaml with a fromDatabase reference", - "The agent verifies the fix by checking list_deploys or list_logs after applying the change", - "The agent does not suggest unrelated fixes like port binding or memory optimization" - ] - }, - { - "id": 2, - "turns": [ - "My app on Render is having issues. Can you help me debug it?", - "It's a Node.js web service called 'checkout-service', ID srv-xyz789. It deployed successfully but now it's crashing randomly and I'm seeing exit code 137 in the logs." - ], - "expected_output": "After the user clarifies the service details, the agent identifies exit code 137 as an OOM (Out of Memory) kill signal, retrieves memory metrics to confirm memory usage is near or at the limit, and suggests fixes such as optimizing memory usage, processing data in chunks, or upgrading the plan. It may also suggest setting NODE_OPTIONS=--max-old-space-size.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "srv-xyz789", - "name": "checkout-service", - "status": "live", - "type": "web_service", - "plan": "starter" - } - ], - "logs": [ - { - "serviceId": "srv-xyz789", - "level": "error", - "message": "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory", - "timestamp": "2024-01-24T11:00:00Z" - }, - { - "serviceId": "srv-xyz789", - "level": "error", - "message": "Killed (exit code 137)", - "timestamp": "2024-01-24T11:00:01Z" - } - ], - "metrics": [ - { - "resourceId": "srv-xyz789", - "metricType": "memory_usage", - "values": [ - 480, - 490, - 505, - 512, - 512 - ] - }, - { - "resourceId": "srv-xyz789", - "metricType": "memory_limit", - "values": [ - 512, - 512, - 512, - 512, - 512 - ] - } - ] - }, - "overrides": { - "tool:list_services": { - "responseFromState": "services" - }, - "tool:get_service": { - "responseFromState": "services[serviceId]" - }, - "tool:list_logs": { - "responseFromState": "logs" - }, - "tool:get_metrics": { - "responseFromState": "metrics" - } - } - } - }, - "assertions": [ - "In the first turn, the agent asks a clarifying question to identify which service is having issues", - "After the user provides the service ID and symptoms, the agent calls list_logs to retrieve error logs for srv-xyz789", - "The agent correctly identifies exit code 137 as an OOM (Out of Memory) kill, not a generic crash", - "The agent calls get_metrics with metricTypes memory_usage and memory_limit to confirm memory is at the 512MB limit", - "The agent recommends at least one concrete fix: upgrading the plan, optimizing memory usage, or setting NODE_OPTIONS=--max-old-space-size" - ] - }, - { - "id": 3, - "prompt": "My Render service 'backend-api' (srv-def456) is running but super slow — p95 latency is terrible and users are complaining. I suspect it might be the database. Can you investigate?", - "expected_output": "The agent checks HTTP latency metrics, then investigates the database by listing Postgres instances, checking active connections, and querying pg_stat_statements for slow queries. It identifies the root cause (e.g., slow queries, missing indexes, or connection exhaustion) and suggests targeted fixes like adding indexes or implementing connection pooling.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "srv-def456", - "name": "backend-api", - "status": "live", - "type": "web_service" - } - ], - "postgres": [ - { - "id": "pg-001", - "name": "backend-db", - "status": "available", - "plan": "basic" - } - ], - "metrics": [ - { - "resourceId": "srv-def456", - "metricType": "http_latency", - "quantile": 0.95, - "values": [ - 1800, - 2100, - 1950, - 2300, - 2050 - ] - }, - { - "resourceId": "pg-001", - "metricType": "active_connections", - "values": [ - 90, - 93, - 95, - 96, - 97 - ] - } - ], - "queryResults": [ - { - "postgresId": "pg-001", - "query": "SELECT query, calls, mean_exec_time FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10", - "rows": [ - { - "query": "SELECT * FROM orders WHERE user_id = $1", - "calls": 5000, - "mean_exec_time": 850.5 - }, - { - "query": "SELECT * FROM products WHERE category = $1", - "calls": 3000, - "mean_exec_time": 620.2 - } - ] - } - ] - }, - "overrides": { - "tool:list_services": { - "responseFromState": "services" - }, - "tool:get_service": { - "responseFromState": "services[serviceId]" - }, - "tool:list_postgres_instances": { - "responseFromState": "postgres" - }, - "tool:get_metrics": { - "responseFromState": "metrics" - }, - "tool:query_render_postgres": { - "responseFromState": "queryResults" - } - } - } - }, - "assertions": [ - "The agent calls get_metrics with http_latency at p95 quantile to confirm high latency on srv-def456", - "The agent calls list_postgres_instances to find the associated database", - "The agent calls get_metrics with active_connections for pg-001 and identifies connections are near the 97-connection limit for the Basic plan", - "The agent calls query_render_postgres with a slow query analysis SQL (e.g., pg_stat_statements) to identify slow queries", - "The agent recommends at least one specific fix such as adding an index on orders.user_id or implementing connection pooling", - "The agent does not conflate the database performance issue with unrelated errors like missing env vars or port binding" - ] - } - ] -} diff --git a/plugins/render/skills/render-deploy/evals.json b/plugins/render/skills/render-deploy/evals.json deleted file mode 100644 index fb49279..0000000 --- a/plugins/render/skills/render-deploy/evals.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "skill_name": "render-deploy", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "turns": [ - "I want to deploy my app to Render. It's a Node.js Express API with a PostgreSQL database and a Redis-backed job queue with a Celery-style worker.", - "Yes, deploy from Git repo. Render should provision everything — the web service, Postgres, Redis, and the worker. Repo is https://github.com/acme/api-service on main branch." - ], - "expected_output": "Agent asks clarifying questions about Git vs Docker and infra needs, then generates a complete render.yaml Blueprint with a web service, worker service, PostgreSQL database, and Redis key-value store. It provides a Dashboard deeplink and next-steps checklist.", - "files": [], - "integrations": {}, - "assertions": [ - "Agent asks at least one clarifying question before generating the Blueprint (e.g., Git vs Docker, or infra provisioning scope)", - "Generated render.yaml includes a type:web service with runtime:node, buildCommand using npm ci, and a startCommand", - "Generated render.yaml includes a type:worker service for the job queue processor", - "Generated render.yaml includes a PostgreSQL database block and a Redis/key-value block, both with plan:free", - "The web service references DATABASE_URL and REDIS_URL via fromDatabase connectionString properties", - "Agent provides a Dashboard deeplink in the format https://dashboard.render.com/blueprint/new?repo=https://github.com/acme/api-service", - "Agent instructs the user to commit and push render.yaml before clicking the deeplink" - ] - }, - { - "id": 2, - "prompt": "I have a simple React SPA (Vite build, outputs to dist/) at https://github.com/jdoe/my-portfolio on the main branch. No backend, no database. Deploy it to Render as fast as possible.", - "expected_output": "Agent recognizes this is a single static site with no dependencies, chooses Direct Creation via MCP, calls create_static_site with correct parameters, and verifies the deployment status.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "svc-static-001", - "name": "my-portfolio", - "type": "static_site", - "status": "live", - "serviceDetails": { - "url": "https://my-portfolio.onrender.com" - } - } - ], - "deploys": [ - { - "id": "dep-001", - "serviceId": "svc-static-001", - "status": "live", - "createdAt": "2024-01-15T10:00:00Z" - } - ] - }, - "overrides": { - "tool:list_deploys": { - "responseFromState": "deploys[serviceId]" - }, - "tool:list_services": { - "responseFromState": "services" - } - } - } - }, - "assertions": [ - "Agent selects Direct Creation (MCP) rather than Blueprint, since this is a single static site with no backend or database", - "Agent calls create_static_site (or equivalent MCP tool) with repo set to https://github.com/jdoe/my-portfolio, buildCommand containing 'npm run build', and publishPath of 'dist'", - "Agent does NOT generate a render.yaml file for this single-service static site", - "Agent calls list_deploys to verify deployment status after creation", - "Agent reports the deployed service URL (e.g., https://my-portfolio.onrender.com) to the user" - ] - }, - { - "id": 3, - "prompt": "Generate a render.yaml for my Django app. It uses Postgres, runs DB migrations on deploy, has a daily cron job at 2 AM UTC to clean up old records (node scripts/cleanup.py), needs DJANGO_SECRET_KEY as a secret, and should have a /health endpoint. Repo is git@github.com:myorg/django-app.git", - "expected_output": "Agent generates a complete render.yaml with a Django web service, PostgreSQL database, and cron job. It converts the SSH remote URL to HTTPS for the deeplink, marks DJANGO_SECRET_KEY with sync:false, includes DB migration in the build command, and provides the correct Blueprint deeplink.", - "files": [], - "integrations": {}, - "assertions": [ - "Generated render.yaml includes a type:web service with runtime:python and a buildCommand that runs both pip install and python manage.py migrate", - "Generated render.yaml includes a type:cron service with schedule '0 2 * * *' and a startCommand referencing the cleanup script", - "Generated render.yaml includes a PostgreSQL database block with plan:free and the web service references it via fromDatabase connectionString", - "DJANGO_SECRET_KEY is declared with sync:false (not hardcoded)", - "healthCheckPath is set to /health on the web service", - "Agent converts the SSH remote URL git@github.com:myorg/django-app.git to HTTPS https://github.com/myorg/django-app for the deeplink", - "Agent provides the Blueprint deeplink https://dashboard.render.com/blueprint/new?repo=https://github.com/myorg/django-app" - ] - } - ] -} diff --git a/plugins/render/skills/render-disks/evals.json b/plugins/render/skills/render-disks/evals.json deleted file mode 100644 index 455d7c7..0000000 --- a/plugins/render/skills/render-disks/evals.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "skill_name": "render-disks", - "models": ["claude-sonnet-4-6", "gpt-5.4", "gemini-3.1-pro-preview"], - "evals": [ - { - "id": 1, - "prompt": "I'm running a Node.js web service on Render and users upload profile pictures. Right now uploads disappear after every deploy. How do I fix this and what should my render.yaml look like?", - "expected_output": "The agent explains that Render services have an ephemeral filesystem by default, so file uploads are lost on deploy. It recommends attaching a persistent disk, suggests an appropriate mount path like /opt/render/project/src/uploads, recommends a small starting size (1-5 GB), provides a render.yaml blueprint snippet with the disk configuration, and warns about the single-instance and no zero-downtime deploy constraints.", - "files": [], - "assertions": [ - "The response explains that Render's default filesystem is ephemeral and files are lost on deploy", - "The response recommends attaching a persistent disk and provides a render.yaml snippet with a 'disk' field including mountPath and sizeGB", - "The suggested mount path is a subdirectory like /opt/render/project/src/uploads, NOT /opt/render/project/src itself", - "The response warns that attaching a disk limits the service to a single instance (no horizontal scaling)", - "The response mentions that zero-downtime deploys are disabled when a disk is attached", - "The response suggests starting with a small disk size (e.g., 1-5 GB) since size can be increased but not decreased" - ] - }, - { - "id": 2, - "turns": [ - "I need persistent storage on Render. I want to scale my service to handle more traffic too.", - "So what are my alternatives if I need both persistent file storage and horizontal scaling?" - ], - "expected_output": "In the first turn, the agent explains the fundamental conflict: persistent disks require single-instance mode, so horizontal scaling and autoscaling are not possible with a disk attached. In the second turn, the agent suggests alternatives like external object storage (AWS S3, Cloudflare R2) for files, Render Postgres or Key Value for databases, which allow the service to remain stateless and scale horizontally.", - "files": [], - "assertions": [ - "In the first response, the agent clearly states that persistent disks restrict the service to a single instance and prevent horizontal scaling", - "In the first response, the agent identifies the conflict between the user's two goals (persistent storage + scaling)", - "In the second response, the agent recommends external object storage (e.g., AWS S3 or Cloudflare R2) as an alternative for file storage that allows scaling", - "In the second response, the agent mentions Render Postgres or Render Key Value as managed alternatives that don't require a disk", - "The agent does not suggest that both a persistent disk and horizontal scaling can work together on the same service" - ] - }, - { - "id": 3, - "turns": [ - "My self-managed MySQL on Render had some data corruption. I want to restore from a disk snapshot. Is that the right approach?", - "Got it. So what backup approach should I use instead for my MySQL on Render?" - ], - "expected_output": "In the first turn, the agent strongly advises against using disk snapshots for database recovery, explaining that snapshots may capture a corrupted or inconsistent database state. In the second turn, the agent recommends using database-native backup tools like mysqldump for consistent backups and explains how to transfer backup files off the service using SCP or magic-wormhole.", - "files": [], - "assertions": [ - "The agent explicitly warns against using disk snapshots for database recovery", - "The agent explains that disk snapshots may capture a corrupted or inconsistent database state", - "The agent recommends mysqldump or other database-native backup tools as the correct approach", - "The agent mentions at least one method for transferring backup files (SCP or magic-wormhole)", - "The response mentions that disk snapshot restores are full restores only and all changes after the snapshot are permanently lost" - ] - } - ] -} diff --git a/plugins/render/skills/render-docker/evals.json b/plugins/render/skills/render-docker/evals.json deleted file mode 100644 index 1889157..0000000 --- a/plugins/render/skills/render-docker/evals.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "skill_name": "render-docker", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I need a production-ready multi-stage Dockerfile for a Node.js 20 app. It uses TypeScript, builds to dist/, and should run as a non-root user. The app listens on Render's PORT. Also give me the render.yaml snippet for it.", - "expected_output": "A multi-stage Dockerfile with a builder stage (node:20-bookworm-slim) that runs npm ci and npm run build, and a minimal runner stage (node:20-alpine) that copies only dist/ and node_modules, sets a non-root user, exposes $PORT, and uses CMD [\"node\", \"dist/server.js\"]. Plus a render.yaml snippet with runtime: docker, dockerfilePath, and dockerContext fields.", - "files": [], - "integrations": {}, - "assertions": [ - "The Dockerfile uses at least two FROM stages (builder and runner)", - "The runner stage uses a minimal base image such as node:20-alpine", - "A non-root user is created and set via USER directive in the runner stage", - "The CMD or ENTRYPOINT in the runner stage starts the app from dist/", - "The render.yaml snippet includes runtime: docker, dockerfilePath, and dockerContext fields", - "PORT is exposed and bound to 0.0.0.0 or referenced via $PORT" - ] - }, - { - "id": 2, - "turns": [ - "I want to deploy a prebuilt image from a private registry on Render. Can you help me set that up?", - "It's on GHCR — ghcr.io/myorg/api:v2.1.0. I already created a credential in the Render Dashboard called 'ghcr-token'." - ], - "expected_output": "The agent first asks which registry and whether credentials are already set up, then provides a render.yaml snippet using runtime: image with image.url set to ghcr.io/myorg/api:v2.1.0 and registryCredential.fromRegistryCreds.name set to 'ghcr-token'. It also warns that prebuilt image services do not auto-deploy on tag updates and recommends using a deploy hook or manual redeploy.", - "files": [], - "integrations": {}, - "assertions": [ - "After the user provides registry details, the agent outputs a render.yaml snippet with runtime: image", - "The image.url is set to ghcr.io/myorg/api:v2.1.0", - "The registryCredential.fromRegistryCreds.name is set to 'ghcr-token'", - "The agent warns that mutable tags like latest or tag-based images do not trigger auto-deploys", - "The agent mentions using a deploy hook or manual redeploy to update the running service" - ] - }, - { - "id": 3, - "prompt": "My Docker builds on Render are really slow — reinstalling all npm packages every time even when I haven't changed them. How do I fix layer ordering and use BuildKit cache mounts? Show me a concrete Dockerfile example.", - "expected_output": "An explanation that dependency manifests (package.json, package-lock.json) must be copied before application source so the install layer is cached when only source changes. A Dockerfile example showing COPY package.json package-lock.json ./ before RUN npm ci, then COPY . . afterward. Optionally demonstrates RUN --mount=type=cache for npm cache. Also mentions .dockerignore excluding node_modules.", - "files": [], - "integrations": {}, - "assertions": [ - "The response explains that copying dependency manifests before source code maximizes layer cache hits", - "The Dockerfile example shows COPY package.json package-lock.json ./ appearing before COPY . .", - "RUN npm ci (or npm install) appears after the manifest COPY and before the source COPY", - "The response mentions or demonstrates RUN --mount=type=cache for npm or pip caches using BuildKit", - "The response recommends a .dockerignore that excludes node_modules to reduce context upload size", - "The syntax=docker/dockerfile:1 directive is included or mentioned to enable BuildKit features" - ] - } - ] -} diff --git a/plugins/render/skills/render-domains/evals.json b/plugins/render/skills/render-domains/evals.json deleted file mode 100644 index dcd4e34..0000000 --- a/plugins/render/skills/render-domains/evals.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "skill_name": "render-domains", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "turns": [ - "I want to add a custom domain to my Render service. Can you help me set it up?", - "It's an apex domain, example.com, and I'm using Cloudflare for DNS. It's a web service called my-app." - ], - "expected_output": "The agent should guide the user to add the domain in the Render Dashboard under Settings > Custom Domains, then configure a CNAME record in Cloudflare with Name '@' and Target 'my-app.onrender.com' (Cloudflare auto-flattens at apex). It should mention that adding the apex also auto-adds www with a redirect, recommend DNS only (gray cloud) during setup, and note that SSL mode should be Full (strict).", - "files": [], - "assertions": [ - "After the user's first message, the agent asks a clarifying question about the domain type (apex vs subdomain), DNS provider, or service type before giving full instructions", - "The agent recommends using a CNAME record with Cloudflare's automatic flattening for the apex domain", - "The agent specifies the CNAME target should be 'my-app.onrender.com' or '.onrender.com'", - "The agent mentions setting Cloudflare SSL mode to 'Full (strict)' to avoid redirect loops", - "The agent mentions that adding the apex domain automatically adds www.example.com with a redirect", - "The agent recommends DNS only (gray cloud) during initial setup in Cloudflare" - ] - }, - { - "id": 2, - "prompt": "I set up a wildcard domain *.example.com on Render but the TLS certificate won't issue. I added the wildcard CNAME record pointing to my service. What am I missing?", - "expected_output": "The agent should explain that wildcard domains require three CNAME records, not just one. It should list the _acme-challenge and _cf-custom-hostname records needed for certificate validation. It should also mention checking for CAA records that might block Let's Encrypt or Google Trust Services.", - "files": [], - "assertions": [ - "The agent mentions that wildcard domains require three CNAME records total", - "The agent specifies the _acme-challenge CNAME record pointing to .verify.renderdns.com for Let's Encrypt validation", - "The agent specifies the _cf-custom-hostname CNAME record pointing to .hostname.renderdns.com for Cloudflare DDoS validation", - "The agent mentions checking for CAA records and adding entries for letsencrypt.org and pki.goog if CAA records exist", - "The agent does NOT suggest the issue is simply DNS propagation delay as the primary cause" - ] - }, - { - "id": 3, - "prompt": "After adding my custom domain on Render, I'm getting ERR_SSL_VERSION_OR_CIPHER_MISMATCH in the browser. I'm using Namecheap DNS. I also have some AAAA records from my old hosting provider. What's going on?", - "expected_output": "The agent should identify the AAAA records as the likely cause. Render is IPv4 only, and stale AAAA records cause some clients to attempt IPv6 connections that fail with SSL errors. The fix is to remove all AAAA records for the domain.", - "files": [], - "assertions": [ - "The agent identifies the AAAA (IPv6) records as the likely cause of the ERR_SSL_VERSION_OR_CIPHER_MISMATCH error", - "The agent explains that Render uses IPv4 only and AAAA records cause clients to attempt IPv6 connections that fail", - "The agent instructs the user to remove all AAAA records for the domain", - "The agent does not incorrectly suggest the issue is a missing TLS certificate or CAA record problem as the primary diagnosis" - ] - } - ] -} diff --git a/plugins/render/skills/render-env-vars/evals.json b/plugins/render/skills/render-env-vars/evals.json deleted file mode 100644 index 3bb73b4..0000000 --- a/plugins/render/skills/render-env-vars/evals.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "skill_name": "render-env-vars", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I need to write a render.yaml for a Node.js web service called 'api' that connects to a Postgres DB named 'prod-db' and a Redis key-value store named 'sessions-kv'. It also needs a generated signing secret, a Stripe API key I'll enter manually, and should pull shared vars from an env group called 'baseline'. Show me the full envVars block.", - "expected_output": "A complete render.yaml envVars block for the 'api' service using fromDatabase (connectionString from prod-db), fromService (connectionString from sessions-kv of type keyvalue), generateValue for the signing secret, sync: false for the Stripe key, and fromGroup for baseline. All wiring patterns should be syntactically correct.", - "files": [], - "integrations": {}, - "assertions": [ - "The response includes a fromDatabase entry referencing 'prod-db' with property: connectionString", - "The response includes a fromService entry referencing 'sessions-kv' with type: keyvalue and property: connectionString", - "The response includes a generateValue: true entry for a signing secret or similar key", - "The response includes a sync: false entry for the Stripe API key", - "The response includes a fromGroup entry referencing 'baseline'", - "The YAML syntax is valid and all envVars entries use correct field names (key, fromDatabase, fromService, generateValue, sync, fromGroup)" - ] - }, - { - "id": 2, - "turns": [ - "My Python web service on Render keeps crashing because it can't bind to the port. I set PORT=8080 in my start command but it's not working.", - "I'm using a native Python runtime, not Docker. My start command is: gunicorn app:app --bind 0.0.0.0:8080" - ], - "expected_output": "The agent explains that Render injects PORT=10000 by default and that Gunicorn on native Python builds often has GUNICORN_CMD_ARGS set to bind 0.0.0.0:10000. It advises the user to either remove the hardcoded 8080 and use $PORT instead, or override GUNICORN_CMD_ARGS/PORT to match. It should clarify that hardcoding a port that conflicts with Render's injected PORT causes the health check to fail.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent asks a clarifying question in the first turn (e.g., about runtime type, Docker vs native, or start command) before giving a full answer", - "After the user provides details, the agent mentions that Render's default PORT is 10000", - "The agent explains that native Python builds may have GUNICORN_CMD_ARGS set to bind 0.0.0.0:10000", - "The agent recommends using $PORT in the bind argument instead of hardcoding 8080", - "The agent does not suggest setting PORT=8080 as a service env var without warning about the conflict with Render's injected PORT" - ] - }, - { - "id": 3, - "prompt": "Use the MCP tools to update the environment variables for my service 'svc-abc123'. I want to set NODE_ENV=production, LOG_LEVEL=debug, and add a DATABASE_URL=postgres://user:pass@host/db.", - "expected_output": "The agent calls the update_environment_variables tool for service svc-abc123 with all three variables set. It should warn that DATABASE_URL contains a plaintext password and suggest using fromDatabase wiring or a secret manager instead of committing it directly.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "svc-abc123", - "name": "my-node-app", - "type": "web", - "status": "live" - } - ] - }, - "overrides": { - "GET /services/{serviceId}": { - "responseFromState": "services[serviceId]" - }, - "tool:get_service": { - "responseFromState": "services[serviceId]" - }, - "tool:update_environment_variables": { - "response": { - "success": true, - "message": "Environment variables updated successfully" - } - } - } - } - }, - "assertions": [ - "The agent calls the update_environment_variables tool targeting service ID svc-abc123", - "The tool call includes all three variables: NODE_ENV=production, LOG_LEVEL=debug, and DATABASE_URL", - "The agent warns that the DATABASE_URL contains a plaintext password in the value", - "The agent suggests an alternative such as using fromDatabase wiring in render.yaml or storing the secret via the Dashboard instead of passing it as a plain value", - "The agent confirms or reports the result of the update_environment_variables call" - ] - } - ] -} diff --git a/plugins/render/skills/render-keyvalue/evals.json b/plugins/render/skills/render-keyvalue/evals.json deleted file mode 100644 index ef4a58b..0000000 --- a/plugins/render/skills/render-keyvalue/evals.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "skill_name": "render-keyvalue", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I'm writing a render.yaml Blueprint for my Django app. I need a Key Value instance for caching and want to wire it to my web service as REDIS_URL. Can you give me the Blueprint YAML for the Key Value service and the envVar wiring?", - "expected_output": "The agent provides a Blueprint YAML snippet with a keyvalue service using allkeys-lru for caching, includes ipAllowList, and shows the fromService envVar wiring with type: keyvalue and property: connectionString.", - "files": [], - "assertions": [ - "The response includes a YAML block with `type: keyvalue`", - "The YAML includes `ipAllowList` (e.g., `ipAllowList: []`)", - "The maxmemoryPolicy is set to `allkeys-lru` since the use case is caching", - "The envVar wiring uses `fromService` with `type: keyvalue` and `property: connectionString`", - "The envVar key is `REDIS_URL` or the agent explains how to name it appropriately" - ] - }, - { - "id": 2, - "turns": [ - "I need to add Redis to my Render app for background jobs. What maxmemory policy should I use?", - "I'm using Sidekiq with Ruby on Rails. The queue processes payment webhooks so I absolutely cannot lose jobs." - ], - "expected_output": "The agent first asks about the use case or queue framework, then after learning it's Sidekiq for payment webhooks, recommends noeviction policy and explains why allkeys-lru would be dangerous for job queues. May also provide Sidekiq connection example.", - "files": [], - "assertions": [ - "In the first turn, the agent either asks a clarifying question about the use case/framework or explains the tradeoff between noeviction and allkeys-lru", - "After learning about Sidekiq and payment webhooks, the agent recommends `noeviction` as the maxmemory policy", - "The agent explains that `allkeys-lru` would evict queued jobs, which is unacceptable for payment processing", - "The agent mentions or shows how to configure Sidekiq with REDIS_URL or provides relevant connection guidance" - ] - }, - { - "id": 3, - "prompt": "I'm getting 'OOM command not allowed' errors on my Render Key Value instance. It's used as a Celery broker for task queues. How do I fix this?", - "expected_output": "The agent explains the error means memory is full with noeviction policy, and recommends upgrading to a larger instance or adding TTLs. It should NOT recommend switching to allkeys-lru since this is a job queue. It should also mention that downgrades are not possible.", - "files": [], - "assertions": [ - "The response explains that the OOM error occurs because the instance memory is full and the maxmemoryPolicy is `noeviction`", - "The agent recommends upgrading to a larger instance type as a primary fix", - "The agent does NOT recommend switching to `allkeys-lru` as the primary solution, since the use case is a Celery task queue where data loss is unacceptable", - "The agent suggests adding TTLs to keys or reviewing data patterns as additional mitigation strategies", - "The response mentions that instance type downgrades are not possible (upgrade-only)" - ] - } - ] -} diff --git a/plugins/render/skills/render-mcp/evals.json b/plugins/render/skills/render-mcp/evals.json deleted file mode 100644 index 93630bc..0000000 --- a/plugins/render/skills/render-mcp/evals.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "skill_name": "render-mcp", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "turns": [ - "I want to connect my AI coding tool to Render so I can manage my services. How do I set up MCP?", - "I'm using Cursor." - ], - "expected_output": "The agent should first ask which AI tool the user is using, then provide Cursor-specific setup instructions including the mcp.json config file path, the correct URL, Bearer token auth, and verification steps.", - "files": [], - "assertions": [ - "After the user says Cursor, the agent provides the config file path ~/.cursor/mcp.json", - "The response includes the MCP URL https://mcp.render.com/mcp", - "The response shows the JSON config structure with mcpServers.render containing url and headers with Authorization Bearer token", - "The agent mentions getting an API key from the Render Dashboard settings page", - "The agent recommends restarting Cursor and verifying with list_services()", - "The agent's first message asks which AI tool the user is using before providing tool-specific instructions" - ] - }, - { - "id": 2, - "prompt": "I'm getting a 401 Unauthorized error when I try to use list_services() in Claude Code. The MCP server was working yesterday. How do I fix this?", - "expected_output": "The agent should diagnose the 401 as an authentication issue, suggest generating a new API key, explain how to update the MCP config in Claude Code using the CLI commands, and recommend restarting and verifying.", - "files": [], - "assertions": [ - "The response identifies the 401 error as related to a missing, invalid, or expired API key", - "The response includes the URL to generate a new API key: https://dashboard.render.com/u/*/settings#api-keys", - "The response explains how to update the MCP server in Claude Code using 'claude mcp remove render' and then re-adding with 'claude mcp add --transport http render https://mcp.render.com/mcp --header \"Authorization: Bearer \"'", - "The response advises restarting Claude Code after updating the config", - "The response suggests verifying the fix by running list_services()" - ] - }, - { - "id": 3, - "prompt": "I set up Render MCP in Codex but list_services() returns empty results even though I have services in my Render dashboard. What's going on?", - "expected_output": "The agent should identify this as likely a workspace selection issue, explain how to check and switch workspaces using MCP tools, and also mention verifying the RENDER_API_KEY env var is set correctly for Codex.", - "files": [], - "assertions": [ - "The response suggests checking the active workspace with get_selected_workspace()", - "The response suggests listing available workspaces with list_workspaces() to find the correct one", - "The response explains that all MCP operations run against the active workspace and the wrong workspace may be selected", - "The response mentions verifying that the RENDER_API_KEY environment variable is set in the shell where Codex runs", - "The response does not suggest the MCP server URL is wrong as the primary cause since the connection itself is working" - ] - } - ] -} diff --git a/plugins/render/skills/render-migrate-from-heroku/evals.json b/plugins/render/skills/render-migrate-from-heroku/evals.json deleted file mode 100644 index 06cd168..0000000 --- a/plugins/render/skills/render-migrate-from-heroku/evals.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "skill_name": "render-migrate-from-heroku", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I want to migrate my Heroku app to Render. Here's my project: Procfile has `web: gunicorn myapp.wsgi` and `worker: celery -A myapp worker`. I have requirements.txt with Django and celery. Heroku app uses Standard-2X dynos, heroku-postgresql:essential-2, and heroku-redis:premium-3. Region is US. I have 14 config vars. Please generate the render.yaml Blueprint.", - "expected_output": "The agent should generate a render.yaml using the projects/environments pattern with a web service, worker service, Postgres database, and Key Value store. Plans should be mapped correctly: Standard-2X → standard, essential-2 → basic-1gb with diskSizeGB 32, premium-3 → standard. Region should be oregon. Secrets should use sync: false. DATABASE_URL should use fromDatabase and REDIS_URL should use fromService.", - "files": [], - "assertions": [ - "The generated YAML starts with 'projects:' key and uses the projects/environments pattern, not flat top-level services", - "The web service has plan 'standard' (mapped from Standard-2X dynos) and runtime 'python'", - "A worker service of type 'worker' is included with the celery start command", - "The Postgres database has plan 'basic-1gb' and diskSizeGB set to 32 (from essential-2)", - "The Key Value store has plan 'standard' (mapped from premium-3) and includes ipAllowList", - "DATABASE_URL uses fromDatabase and REDIS_URL uses fromService with type keyvalue — no hardcoded connection strings" - ] - }, - { - "id": 2, - "turns": [ - "I want to move off Heroku to Render. Can you help me migrate?", - "It's a Node.js app with just a web process: `web: npm start`. No workers, no database, no Redis. I'm on Heroku's Basic dyno in the US region. The repo is at git@github.com:acme/myapp.git" - ], - "expected_output": "After clarification, the agent should identify this as a single-service migration eligible for MCP Direct Creation (no workers, no databases). It should map Basic dyno to starter plan, use node runtime, set region to oregon, convert the SSH git URL to HTTPS format, and present a migration plan before creating anything.", - "files": [], - "assertions": [ - "The agent asks clarifying questions in the first turn about the project details (runtime, Procfile, databases, dyno size, etc.) before proceeding", - "The agent identifies this as eligible for MCP Direct Creation since it's a single web service with no workers, databases, or Key Value stores", - "The Heroku Basic dyno is mapped to Render plan 'starter'", - "The SSH URL git@github.com:acme/myapp.git is converted to HTTPS format https://github.com/acme/myapp", - "The agent presents a migration plan summary and asks for user confirmation before creating any resources" - ] - }, - { - "id": 3, - "turns": [ - "I'm migrating from Heroku to Render. My app uses heroku-postgresql:standard-4 and Performance-M dynos. I don't have Heroku MCP set up. How do I handle the data migration?", - "The pg:info shows Data Size is 45 GB. The Render Postgres is already provisioned with ID dpg-xyz789. I have the Render CLI installed." - ], - "expected_output": "The agent should recommend the traditional pg_dump/pg_restore approach for a 45 GB database (over 2 GB threshold). It should map standard-4 to accelerated-32gb with diskSizeGB 770. It should instruct the user to get the Render external connection string from the Dashboard, put Heroku in maintenance mode, and provide the dump/restore commands. It should also mention data validation steps.", - "files": [], - "assertions": [ - "The agent maps heroku-postgresql:standard-4 to Render plan 'accelerated-32gb' with diskSizeGB of 770", - "The agent recommends the traditional pg_dump -Fc / pg_restore approach (not render psql) since the database is 45 GB, which exceeds the 2 GB threshold", - "The agent instructs the user to get the Render external connection string from the Dashboard link https://dashboard.render.com/d/dpg-xyz789", - "The agent includes the step to put Heroku in maintenance mode before starting the dump", - "The agent mentions data validation after restore, including comparing row counts between Heroku and Render", - "The agent notes that storage is billed separately on Render at $0.30/GB/month and advises right-sizing based on actual usage vs Heroku allocation" - ] - } - ] -} diff --git a/plugins/render/skills/render-monitor/evals.json b/plugins/render/skills/render-monitor/evals.json deleted file mode 100644 index 07cb2d7..0000000 --- a/plugins/render/skills/render-monitor/evals.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "skill_name": "render-monitor", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "Run a health check on my Render service `movie-mcp-server`. I want to see its status, latest deployment, and any recent errors. If CPU/memory or HTTP latency metrics aren't available in this environment, explain that clearly and tell me what tool or command you'd need.", - "expected_output": "The agent identifies the named service, checks its status, inspects the latest deployment, and checks for recent error logs. If metrics are unavailable because MCP tools are missing, it says so explicitly instead of hallucinating data and gives a concrete next step.", - "files": [], - "assertions": [ - "The agent enumerates or looks up services to identify `movie-mcp-server`", - "If the exact service name is not found, the agent clearly says so and asks for confirmation before switching to a different service", - "The agent checks the latest deployment for the target service", - "The agent checks recent logs for error-level issues on the target service", - "The agent does not invent CPU, memory, or latency metrics if they are unavailable in the current environment", - "If metrics are unavailable, the agent explicitly explains that Render MCP is required for CPU, memory, or HTTP latency checks" - ] - }, - { - "id": 2, - "turns": [ - "My Render web service `api-prod` has been slow for about the last 2 hours. Investigate what you can from this environment: check the service status, recent deploys, and logs around that slowdown window.", - "The slowdown seems worst on `/api/orders`. If endpoint-specific latency metrics aren't available, tell me what tool you'd need and what you'd check next." - ], - "expected_output": "The agent can begin investigating from turn 1 because the service name and slowdown window are already provided. If turn 2 is consumed, it should incorporate the `/api/orders` detail and explain whether endpoint-specific latency metrics require MCP. The eval should still reward solid investigation even if the second turn is not required to unlock the task.", - "files": [], - "assertions": [ - "The agent investigates the specific service `api-prod` rather than staying generic", - "The agent checks logs covering roughly the last 2 hours or clearly attempts to scope the log inspection to the reported slowdown window", - "The agent checks the latest deploy or service status to rule out an obvious deploy-related regression", - "If the `/api/orders` detail is consumed or endpoint-specific latency is discussed, the agent says MCP is required rather than pretending it retrieved endpoint latency metrics" - ] - }, - { - "id": 3, - "prompt": "Check the health of my Render PostgreSQL database. I'm worried about connection counts and slow queries. The postgres ID is `postgres-abc123`. If you can't access database metrics or run SQL from this environment, say exactly what is blocked and what command or MCP tool you'd use next.", - "expected_output": "The agent first verifies whether the supplied Postgres resource exists. If it cannot access database metrics or execute SQL in the current environment, it explains the limitation clearly, avoids inventing findings, and provides concrete next-step commands or MCP tools for checking connection counts and slow queries.", - "files": [], - "assertions": [ - "The agent first attempts to verify whether `postgres-abc123` exists rather than assuming it is valid", - "If the database cannot be found or accessed, the agent says so clearly instead of fabricating health data", - "The agent explains that active connection metrics require Render metrics access or MCP support", - "The agent explains that slow-query inspection requires SQL access such as `query_render_postgres` or `render psql`", - "The agent gives actionable next steps for checking connection counts or slow queries once the right access is available" - ] - } - ] -} diff --git a/plugins/render/skills/render-networking/evals.json b/plugins/render/skills/render-networking/evals.json deleted file mode 100644 index 116ac3a..0000000 --- a/plugins/render/skills/render-networking/evals.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "skill_name": "render-networking", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I have a public API gateway (Web Service) and a backend analytics service (Private Service) in the same Render workspace and region. How do I wire them together so the gateway can call the analytics service privately? What URL format should I use in my gateway code?", - "expected_output": "The agent explains the gateway pattern: use the analytics service's internal hostname and port from Dashboard > Connect > Internal tab. Provides the URL template http://[internal-hostname]:[port]/path, warns about explicit scheme requirement, and notes both services must be in the same region and workspace. Mentions keeping the Private Service off the public internet.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent instructs the user to find the internal hostname via Dashboard > Connect > Internal tab", - "The agent provides the URL template in the form http://[internal-hostname]:[port]/path", - "The agent explicitly warns that an http:// or https:// scheme prefix is required and a bare hostname may not work", - "The agent confirms both services must be in the same region and same workspace for private DNS to work", - "The agent describes the gateway pattern: public Web Service calls Private Service over the private network" - ] - }, - { - "id": 2, - "turns": [ - "My background worker can't connect to my other services privately. I set up internal hostnames but nothing works.", - "I'm trying to call my payment processing service from the worker using its internal hostname. The worker is a Background Worker type on Render." - ], - "expected_output": "After learning the service type is a Background Worker, the agent explains that Background Workers have no internal hostname and cannot receive inbound private connections, but CAN make outbound connections. The agent clarifies the worker should connect TO the payment service using the payment service's internal URL, not the other way around. If the payment service is a Private Service or Web Service, the worker can reach it outbound.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent asks a clarifying question in the first turn to understand the service types or direction of traffic", - "After learning the service is a Background Worker, the agent explains Background Workers cannot receive inbound private connections", - "The agent clarifies that Background Workers CAN make outbound private connections to other services", - "The agent explains the worker should use the payment service's internal URL (from Connect > Internal) to call it outbound", - "The agent does not suggest pointing other services at a worker internal hostname, since workers have none" - ] - }, - { - "id": 3, - "prompt": "I have a web service scaled to 5 instances and need to implement custom load balancing in my app — sometimes I need to send a request to a specific instance. How does Render's service discovery work for this?", - "expected_output": "The agent explains the discovery DNS pattern: the [hostname]-discovery name resolves to all instance IPs. Mentions the RENDER_DISCOVERY_SERVICE environment variable. Explains this allows app code to enumerate instances for custom selection, retries, or fan-out instead of relying on the single internal A-record. Contrasts with the standard internal hostname which routes to one instance.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent explains the [hostname]-discovery DNS name that resolves to all instance IPs", - "The agent mentions the RENDER_DISCOVERY_SERVICE environment variable", - "The agent contrasts the discovery hostname (all instances) with the standard internal hostname (single routing target)", - "The agent describes valid use cases: custom load balancing, health aggregation, or per-instance fan-out logic", - "The agent notes this is useful when round-robin or sticky behavior must be implemented in application code" - ] - } - ] -} diff --git a/plugins/render/skills/render-postgres/evals.json b/plugins/render/skills/render-postgres/evals.json deleted file mode 100644 index b684a20..0000000 --- a/plugins/render/skills/render-postgres/evals.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "skill_name": "render-postgres", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "My Node.js app on Render keeps getting 'too many connections' errors against my Render Postgres DB. The DB has 4 GB RAM. What's my connection limit and how do I fix this?", - "expected_output": "The agent explains that a 4 GB RAM instance (under 8 GB) has a max of 100 connections, that Render has no built-in pooler, and recommends application-side pooling (e.g., pg pool, PgBouncer). It should also mention checking active connections via MCP or pg_stat_activity SQL.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent states the connection limit for under 8 GB RAM is 100 connections", - "The agent explains Render does not provide a built-in connection pooler", - "The agent recommends application-side pooling such as PgBouncer, pgpool, or framework connection pools", - "The agent mentions the query_render_postgres MCP tool or pg_stat_activity SQL as a way to inspect active connections", - "The agent does not suggest upgrading plan as the only solution without first recommending pooling" - ] - }, - { - "id": 2, - "turns": [ - "I want to add read replicas to my Render Postgres database using a Blueprint. Can you help me set that up?", - "The database is called 'my-prod-db' and I want 2 replicas named 'replica-1' and 'replica-2'. I already have one replica named 'old-replica' in production." - ], - "expected_output": "The agent provides a Blueprint snippet using the readReplicas list with 'replica-1' and 'replica-2', and critically warns the user that because 'old-replica' is not in the new list, it will be destroyed when the Blueprint syncs. It should explain that readReplicas is authoritative desired state, not additive.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent provides a valid render.yaml snippet with a readReplicas list containing 'replica-1' and 'replica-2'", - "The agent explicitly warns that 'old-replica' will be destroyed because it is not in the new readReplicas list", - "The agent explains that readReplicas is treated as authoritative desired state, not additive", - "The agent mentions the maximum of 5 read replicas per database", - "The agent asks a clarifying question in the first turn before providing the Blueprint snippet" - ] - }, - { - "id": 3, - "prompt": "I need to check the active connection count on my Render Postgres instance (ID: pg-abc123) and run a quick query to see the top 5 slowest queries.", - "expected_output": "The agent uses the get_metrics MCP tool with resourceId 'pg-abc123' and metricTypes ['active_connections'] to fetch connection metrics, then uses query_render_postgres with the pg_stat_statements SQL to find slow queries. It should note that query_render_postgres is read-only and opens a new connection per query.", - "files": [], - "integrations": { - "render": { - "state": { - "postgres_instances": [ - { - "id": "pg-abc123", - "name": "my-prod-db", - "plan": "pro", - "region": "oregon", - "status": "available" - } - ], - "metrics": [ - { - "id": "pg-abc123", - "metricType": "active_connections", - "values": [ - { - "timestamp": "2024-01-15T10:00:00Z", - "value": 45 - }, - { - "timestamp": "2024-01-15T10:05:00Z", - "value": 62 - }, - { - "timestamp": "2024-01-15T10:10:00Z", - "value": 78 - } - ] - } - ] - }, - "overrides": { - "tool:get_metrics": { - "responseFromState": "metrics[id]" - }, - "tool:get_postgres": { - "responseFromState": "postgres_instances[id]" - }, - "tool:query_render_postgres": { - "response": { - "rows": [ - { - "query": "SELECT * FROM orders WHERE ...", - "mean_exec_time": 1523.4, - "calls": 120 - }, - { - "query": "UPDATE users SET last_login ...", - "mean_exec_time": 890.1, - "calls": 450 - }, - { - "query": "SELECT count(*) FROM events ...", - "mean_exec_time": 654.2, - "calls": 30 - }, - { - "query": "INSERT INTO logs ...", - "mean_exec_time": 210.5, - "calls": 9000 - }, - { - "query": "SELECT id, name FROM products ...", - "mean_exec_time": 98.3, - "calls": 5000 - } - ] - } - } - } - } - }, - "assertions": [ - "The agent calls the get_metrics tool with resourceId 'pg-abc123' and metricTypes including 'active_connections'", - "The agent calls query_render_postgres with postgresId 'pg-abc123' and a SQL query referencing pg_stat_statements ordered by mean_exec_time", - "The agent presents the top slow queries from the result, including mean_exec_time values", - "The agent notes that query_render_postgres runs in a read-only transaction and opens a new connection per query", - "The agent does not suggest using query_render_postgres as a substitute for application-level connection pooling" - ] - } - ] -} diff --git a/plugins/render/skills/render-private-services/evals.json b/plugins/render/skills/render-private-services/evals.json deleted file mode 100644 index 761d90e..0000000 --- a/plugins/render/skills/render-private-services/evals.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "skill_name": "render-private-services", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "I need a Render Blueprint (render.yaml) for a microservices architecture: a public Node.js gateway that routes to two internal services — a Python user-service and a Go order-service. The gateway needs env vars pointing to both internal services. All in Oregon region, starter plan.", - "expected_output": "A complete render.yaml with the gateway as type: web and both internal services as type: pserv, with fromService wiring using property: hostport for each internal service URL env var, all in the oregon region on starter plans.", - "files": [], - "assertions": [ - "The output contains a YAML block with a service of type 'web' for the gateway", - "The output contains two services of type 'pserv' — one named user-service and one named order-service", - "The gateway service has envVars using fromService with type: pserv and property: hostport for both internal services", - "All services specify region: oregon and plan: starter", - "The user-service uses runtime: python and the order-service uses runtime: go", - "The gateway service is the only one that would receive public internet traffic" - ] - }, - { - "id": 2, - "turns": [ - "I have a process that handles billing. It receives requests from our main app and also processes jobs from a Redis queue. Should I make it a private service or a background worker on Render?", - "It definitely needs to receive HTTP requests from our gateway for synchronous billing lookups, and it also polls Redis for async invoice generation." - ], - "expected_output": "The agent should recommend a private service (pserv) because the process must bind to a port and receive HTTP requests from other services. It should explain that background workers cannot receive traffic. It may also suggest splitting into two services if desired, but the key recommendation is pserv since it listens on a port.", - "files": [], - "assertions": [ - "The agent asks a clarifying question in the first turn about whether the process needs to receive incoming requests/traffic or only pulls from the queue", - "After the user confirms it receives HTTP requests, the agent recommends using a private service (type: pserv)", - "The agent explains that background workers cannot receive network traffic or bind to a port", - "The agent mentions that the service must bind to 0.0.0.0 (not localhost or 127.0.0.1)", - "The agent references the rule of thumb: if it listens on a port and other services call it, it's a private service" - ] - }, - { - "id": 3, - "prompt": "I'm setting up a gRPC service on Render that only my other Render services need to call. It should listen on port 50051 using Go. How do I configure this and how do consuming services connect to it?", - "expected_output": "The agent should explain how to configure a gRPC private service with type: pserv, binding to 0.0.0.0:50051, and show how consumers wire the address using fromService with property: hostport. It should note that no TLS is needed for internal traffic.", - "files": [], - "assertions": [ - "The output specifies type: pserv for the gRPC service", - "The output instructs binding to 0.0.0.0 on port 50051 (not localhost or 127.0.0.1)", - "The output shows consumer wiring using fromService with type: pserv and property: hostport", - "The output mentions that TLS is not needed for internal private network traffic", - "The output includes a Blueprint YAML snippet or configuration example for the gRPC service with runtime: go", - "The output explains that consumers connect via : (e.g., grpc-service:50051) on the private network" - ] - } - ] -} diff --git a/plugins/render/skills/render-scaling/evals.json b/plugins/render/skills/render-scaling/evals.json deleted file mode 100644 index a5c493f..0000000 --- a/plugins/render/skills/render-scaling/evals.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "skill_name": "render-scaling", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "turns": [ - "I want to set up autoscaling for my Render web service. Can you help?", - "It's a stateless Node.js API. We're on a Professional workspace. The service ID is svc-abc123. I want it to scale between 2 and 8 instances, and it's mostly CPU-bound." - ], - "expected_output": "The agent asks clarifying questions about the service type, workspace plan, desired min/max instances, and whether it's CPU or memory bound. After the user provides details, the agent recommends a CPU target around 70%, provides the correct Blueprint YAML with scaling block (minInstances: 2, maxInstances: 8, targetCPUPercent: ~70), and explains how to apply it via Dashboard or Blueprint.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "svc-abc123", - "name": "node-api", - "type": "web_service", - "plan": "standard", - "numInstances": 1 - } - ] - }, - "overrides": { - "GET /services/{serviceId}": { - "responseFromState": "services[serviceId]" - }, - "tool:get_service": { - "responseFromState": "services[serviceId]" - } - } - } - }, - "assertions": [ - "In the first turn, the agent asks at least one clarifying question about the service (e.g., workspace plan, instance count range, or CPU vs memory bound).", - "After the user provides details, the agent provides a Blueprint YAML scaling block with minInstances: 2, maxInstances: 8.", - "The agent recommends a targetCPUPercent value around 70% for a CPU-bound service.", - "The agent does NOT include a conflicting numInstances field alongside the scaling block, or explicitly warns that autoscaling takes precedence over numInstances.", - "The agent mentions that autoscaling requires a Professional or higher workspace." - ] - }, - { - "id": 2, - "prompt": "My Render service svc-xyz789 has a persistent disk attached and I'm trying to scale it to 3 instances but it's not working. What's wrong and what are my options?", - "expected_output": "The agent explains that services with persistent disks cannot scale beyond a single instance on Render. It advises the user to either remove the disk to enable horizontal scaling, or scale vertically by upgrading the instance plan instead.", - "files": [], - "integrations": { - "render": { - "state": { - "services": [ - { - "id": "svc-xyz789", - "name": "disk-service", - "type": "web_service", - "plan": "standard", - "numInstances": 1, - "disk": { - "name": "data-disk", - "sizeGB": 10 - } - } - ] - }, - "overrides": { - "GET /services/{serviceId}": { - "responseFromState": "services[serviceId]" - }, - "tool:get_service": { - "responseFromState": "services[serviceId]" - } - } - } - }, - "assertions": [ - "The agent clearly states that services with a persistent disk attached are limited to a single instance and cannot be horizontally scaled.", - "The agent explains that removing the disk would allow horizontal scaling.", - "The agent suggests vertical scaling (upgrading the plan) as an alternative to handle more load while keeping the disk.", - "The agent does NOT suggest that autoscaling or manual instance count settings can override the single-instance constraint for disk-attached services." - ] - }, - { - "id": 3, - "prompt": "I'm trying to right-size my Render service. It's currently on the pro plan (4 CPU, 8 GB RAM) with 6 instances. CPU averages 15% and memory averages 20%. How can I reduce costs without hurting performance?", - "expected_output": "The agent identifies that the service is significantly over-provisioned based on low CPU and memory utilization. It recommends downsizing the plan (e.g., to standard or standard_plus) and reducing instance count or setting tighter autoscaling targets. It explains the vertical vs horizontal tradeoffs and references per-second billing.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent identifies that 15% CPU and 20% memory utilization across 6 pro instances indicates significant over-provisioning.", - "The agent recommends downgrading the plan to a smaller instance type such as standard (1 CPU, 2 GB) or standard_plus (2 CPU, 4 GB).", - "The agent suggests reducing the number of instances or setting autoscaling targets higher (e.g., CPU target ~70%) to allow scale-down.", - "The agent references that Render uses per-second billing, so reducing instance count and plan size directly reduces cost.", - "The agent recommends monitoring CPU and memory utilization after changes to confirm the right-sized configuration holds under real traffic." - ] - } - ] -} diff --git a/plugins/render/skills/render-static-sites/evals.json b/plugins/render/skills/render-static-sites/evals.json deleted file mode 100644 index 61bcc2d..0000000 --- a/plugins/render/skills/render-static-sites/evals.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "skill_name": "render-static-sites", - "models": ["claude-sonnet-4-6", "gpt-5.4", "gemini-3.1-pro-preview"], - "evals": [ - { - "id": 1, - "prompt": "I want to deploy my frontend app to Render as a static site. Can you help me set up the Blueprint config?", - "expected_output": "The agent should produce a render.yaml Blueprint with type: web, runtime: static, the correct Vite build command and publish path (dist), a rewrite rule for SPA routing, and PR previews set to automatic.", - "files": [], - "assertions": [ - "The agent asks a clarifying question in the first turn about which framework or build tool is being used", - "The final Blueprint uses 'type: web' with 'runtime: static' (not 'type: static')", - "The build command includes 'npm ci && npm run build' or equivalent for Vite", - "The staticPublishPath is set to 'dist'", - "A rewrite rule with source '/*' and destination '/index.html' is included for SPA routing", - "PR previews are configured with 'generation: automatic'" - ] - }, - { - "id": 2, - "prompt": "I'm building a Next.js app and I'm not sure whether to deploy it as a static site or a web service on Render. I use getServerSideProps on some pages but most pages are static. What should I do?", - "expected_output": "The agent should explain that since the app uses getServerSideProps (SSR), it cannot be fully statically exported and should be deployed as a Web Service. It should mention that only if the user can remove SSR and use 'output: export' in next.config.js could they use a static site. It should clearly distinguish the two options.", - "files": [], - "assertions": [ - "The response explains that getServerSideProps requires a running server and thus a Web Service", - "The response mentions that a static site deployment requires 'output: export' in next.config.js", - "The response recommends a Web Service for the user's current setup with SSR pages", - "The response mentions that static sites cannot run server code", - "If a static export path is mentioned, it correctly states the publish path as 'out'" - ] - }, - { - "id": 3, - "prompt": "I have a Gatsby site on Render. I need to add security headers (X-Frame-Options, X-Content-Type-Options) globally, cache static assets for a year, and redirect /old-docs/* to /docs/*. The SPA catch-all should still work. Show me the Blueprint routes and headers config.", - "expected_output": "The agent should provide a Blueprint snippet with correctly ordered routes (specific redirect for /old-docs/* above the SPA catch-all rewrite), security headers on /*, cache headers on /assets/* with immutable, and no-cache on index.html.", - "files": [], - "assertions": [ - "The redirect rule for '/old-docs/*' to '/docs/*' appears before the SPA catch-all rewrite rule in the routes list", - "The SPA catch-all rewrite rule has source '/*' and destination '/index.html'", - "An X-Frame-Options header with value 'DENY' is set on path '/*'", - "An X-Content-Type-Options header with value 'nosniff' is set on path '/*'", - "A Cache-Control header for static assets includes 'max-age=31536000' and 'immutable'" - ] - } - ] -} diff --git a/plugins/render/skills/render-web-services/evals.json b/plugins/render/skills/render-web-services/evals.json deleted file mode 100644 index a694f31..0000000 --- a/plugins/render/skills/render-web-services/evals.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "skill_name": "render-web-services", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "prompt": "My Render web service deploys succeed in the UI but never actually take traffic. I'm running an Express app and set up a health check at /health. Here's my server setup: `app.listen(3000, '127.0.0.1', () => console.log('listening'))`. What's wrong and how do I fix it?", - "expected_output": "The agent identifies two problems: (1) the app is binding to 127.0.0.1 instead of 0.0.0.0, which prevents Render's proxy from reaching it, and (2) the port is hardcoded to 3000 instead of using the PORT environment variable. It provides corrected code using 0.0.0.0 and process.env.PORT, and explains that health check failures from connection refused block deploys from going live.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent identifies binding to 127.0.0.1 as the root cause of the connection refused issue", - "The agent instructs the user to bind to 0.0.0.0 instead of 127.0.0.1", - "The agent instructs the user to use the PORT environment variable (process.env.PORT) instead of hardcoding 3000", - "The agent explains that failed health checks prevent the new deploy from receiving production traffic", - "The agent provides corrected code or a concrete fix example for the listen call" - ] - }, - { - "id": 2, - "turns": [ - "I want to add a custom domain to my Render web service. Can you help me set it up?", - "The service is called 'my-api' and I want to use the apex domain example.com as well as www.example.com. I'm on a paid plan." - ], - "expected_output": "The agent asks for clarification about which domain(s) and service the user wants to configure. After the user provides details, it explains: for www.example.com use a CNAME pointing to my-api.onrender.com; for the apex domain example.com, a plain CNAME at the zone apex is not RFC-compliant so the user needs ALIAS/ANAME/CNAME flattening at their DNS provider or should redirect the apex to www. It also mentions Render auto-provisions TLS certificates after DNS verification, and that both domains can be listed in a Blueprint under the domains field.", - "files": [], - "integrations": {}, - "assertions": [ - "The agent asks a clarifying question in the first turn before providing configuration details", - "After the user provides domain details, the agent explains that www.example.com should use a CNAME record pointing to my-api.onrender.com", - "The agent explains that a plain CNAME at the apex (example.com) is not supported by DNS RFCs and recommends ALIAS, ANAME, or CNAME flattening", - "The agent mentions that Render automatically provisions and renews TLS certificates for verified custom domains", - "The agent provides a Blueprint YAML example or mentions the domains field listing both hostnames" - ] - }, - { - "id": 3, - "prompt": "I'm configuring a Render web service with a persistent disk for file storage. I want zero-downtime deploys, horizontal scaling to 3 instances, and I need to run DB migrations before each deploy. What limitations should I know about and how should I set up the deploy lifecycle?", - "expected_output": "The agent explains: (1) a persistent disk makes zero-downtime deploys unavailable and limits the service to a single instance (no horizontal scaling), so the user cannot have all three features simultaneously with a disk attached. (2) DB migrations should use preDeployCommand which runs in the new image before traffic switches and cancels the deploy if it fails. (3) The disk is not mounted during the build phase. It recommends the user reconsider the architecture if they need scaling (e.g., use external object storage instead of a disk).", - "files": [], - "integrations": {}, - "assertions": [ - "The agent states that a persistent disk disables zero-downtime deploys", - "The agent states that a service with a persistent disk is limited to a single instance and cannot be horizontally scaled", - "The agent explains that preDeployCommand should be used for database migrations and runs before traffic switches to new instances", - "The agent explains that if preDeployCommand fails (exits non-zero), the deploy is canceled and the previous revision continues serving", - "The agent notes that the persistent disk is not mounted during the build phase, only at runtime", - "The agent acknowledges the conflict between the user's goals and suggests reconsidering the architecture (e.g., external storage) to enable scaling and zero-downtime deploys" - ] - } - ] -} diff --git a/plugins/render/skills/render-workflows/evals.json b/plugins/render/skills/render-workflows/evals.json deleted file mode 100644 index 8c68467..0000000 --- a/plugins/render/skills/render-workflows/evals.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "skill_name": "render-workflows", - "models": [ - "claude-sonnet-4-6", - "gpt-5.4", - "gemini-3.1-pro-preview" - ], - "evals": [ - { - "id": 1, - "turns": [ - "I want to set up Render Workflows in my project. Can you help me get started?", - "It's a Python project. I have a pyproject.toml and some .py files already." - ], - "expected_output": "The agent should first ask what language the project uses (or detect it), then recommend `render workflows init` as the primary setup path, explain CLI version requirements (2.11.0+), and provide a verification checklist including starting the local dev server and testing the ping task.", - "files": [], - "assertions": [ - "The agent asks about the project language or detects it before proceeding with scaffolding", - "The agent recommends `render workflows init` as the primary scaffolding method", - "The agent mentions the Render CLI version requirement of 2.11.0+", - "The agent provides verification steps including `render workflows dev -- python workflows/main.py`", - "The agent mentions testing with `render workflows tasks list --local` to verify task registration", - "The agent does NOT suggest using Blueprints or render.yaml for deploying workflows" - ] - }, - { - "id": 2, - "prompt": "I have a Python Render Workflow and I need to process 100 images in parallel. Each image needs resizing and uploading. Show me how to implement a fan-out pattern with proper retry config for the upload step since the storage API is flaky.", - "expected_output": "The agent should provide a fan-out pattern using asyncio.gather with a parent task that dispatches subtasks, configure aggressive retry on the upload task, and use the correct SDK API (Workflows, @app.task decorator, Retry class).", - "files": [], - "assertions": [ - "The code uses `from render_sdk import Workflows, Retry` for imports", - "The code defines an async parent task using `@app.task` that uses `asyncio.gather` for parallel fan-out", - "The upload or flaky-API task has a Retry configuration with max_retries, wait_duration_ms, and backoff_scaling parameters", - "All task return values and arguments are JSON-serializable (no class instances or non-serializable types)", - "The agent mentions or demonstrates that subtask calls must be awaited", - "The code includes `app.start()` in a `if __name__ == \"__main__\":` block" - ] - }, - { - "id": 3, - "prompt": "My TypeScript Render Workflow is deployed but tasks aren't showing up when I run `render workflows tasks list`. Locally they work fine with --local flag. Also, I'm getting 'Task not found' errors when I try to trigger runs from my web service using the SDK client. What's going on?", - "expected_output": "The agent should systematically troubleshoot: check that all task files are imported synchronously in the entry point (no dynamic imports), verify the task identifier format is correct (workflow-slug/task-name), check deployment logs, and verify RENDER_API_KEY is set on the calling service.", - "files": [], - "assertions": [ - "The agent mentions checking that all task files use synchronous `import './task-file'` statements, not dynamic imports, to avoid late registration after auto-start", - "The agent explains the task identifier format as `{workflow-slug}/{task-name}` and suggests verifying it matches what's in the Dashboard", - "The agent recommends checking the deployment logs in the Render Dashboard for errors", - "The agent mentions verifying that `RENDER_API_KEY` is set as an environment variable on the calling web service", - "The agent distinguishes between local (`--local` flag) and remote task listing behavior" - ] - } - ] -}