From e1657457eb487fd132fe0c8c8650b89122bc8ec4 Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Wed, 8 Apr 2026 14:19:30 -0700 Subject: [PATCH 01/25] docs: scaffold new directory structure, remove smoke checks --- .github/workflows/docs-ci.yml | 39 +- docs/astro.config.mjs | 22 +- docs/package.json | 3 +- docs/scripts/validate-docs-smoke.mjs | 113 --- .../create-volume-1.png | Bin .../create-volume-2.png | Bin .../{guides => deploy}/s3-tables-query.png | Bin .../docs/development/docs-maintenance.mdx | 69 -- docs/src/content/docs/development/tracing.mdx | 64 -- .../content/docs/essentials/architecture.mdx | 113 --- .../content/docs/essentials/configuration.mdx | 146 --- .../content/docs/essentials/quick-start.mdx | 126 --- .../content/docs/essentials/runtime-modes.mdx | 86 -- .../src/content/docs/essentials/snowflake.mdx | 134 --- .../docs/essentials/support-matrix.mdx | 52 - .../quick-start-ui.png | Bin docs/src/content/docs/guides/aws-lambda.mdx | 255 ----- docs/src/content/docs/guides/dbt.mdx | 190 ---- .../content/docs/guides/end-to-end-dbt.mdx | 170 ---- docs/src/content/docs/guides/s3-tables.mdx | 241 ----- docs/src/content/docs/guides/self-hosted.mdx | 160 --- .../src/content/docs/guides/snowflake-cli.mdx | 102 -- docs/src/content/docs/guides/snowplow.mdx | 161 --- .../content/docs/guides/troubleshooting.mdx | 139 --- .../architecture.png | Bin .../plans/2026-04-08-docs-restructure.md | 938 ++++++++++++++++++ .../2026-04-08-docs-restructure-design.md | 324 ++++++ 27 files changed, 1279 insertions(+), 2368 deletions(-) delete mode 100644 docs/scripts/validate-docs-smoke.mjs rename docs/src/content/docs/{essentials => deploy}/create-volume-1.png (100%) rename docs/src/content/docs/{essentials => deploy}/create-volume-2.png (100%) rename docs/src/content/docs/{guides => deploy}/s3-tables-query.png (100%) delete mode 100644 docs/src/content/docs/development/docs-maintenance.mdx delete mode 100644 docs/src/content/docs/development/tracing.mdx delete mode 100644 docs/src/content/docs/essentials/architecture.mdx delete mode 100644 docs/src/content/docs/essentials/configuration.mdx delete mode 100644 docs/src/content/docs/essentials/quick-start.mdx delete mode 100644 docs/src/content/docs/essentials/runtime-modes.mdx delete mode 100644 docs/src/content/docs/essentials/snowflake.mdx delete mode 100644 docs/src/content/docs/essentials/support-matrix.mdx rename docs/src/content/docs/{essentials => getting-started}/quick-start-ui.png (100%) delete mode 100644 docs/src/content/docs/guides/aws-lambda.mdx delete mode 100644 docs/src/content/docs/guides/dbt.mdx delete mode 100644 docs/src/content/docs/guides/end-to-end-dbt.mdx delete mode 100644 docs/src/content/docs/guides/s3-tables.mdx delete mode 100644 docs/src/content/docs/guides/self-hosted.mdx delete mode 100644 docs/src/content/docs/guides/snowflake-cli.mdx delete mode 100644 docs/src/content/docs/guides/snowplow.mdx delete mode 100644 docs/src/content/docs/guides/troubleshooting.mdx rename docs/src/content/docs/{essentials => reference}/architecture.png (100%) create mode 100644 docs/superpowers/plans/2026-04-08-docs-restructure.md create mode 100644 docs/superpowers/specs/2026-04-08-docs-restructure-design.md diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index a54f928b1..27b28597a 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -51,47 +51,10 @@ jobs: working-directory: ./docs run: pnpm prettier --check . - smoke: - name: Docs Smoke Checks - runs-on: ubuntu-latest - needs: [format] - steps: - - uses: actions/checkout@v6 - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version: "22" - - name: Setup PNPM - uses: pnpm/action-setup@v4 - with: - version: "10.11.0" - run_install: false - - name: Get pnpm store directory - id: pnpm-cache - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT - - name: Setup pnpm cache - uses: actions/cache@v4 - with: - path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/docs/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - name: Install dependencies - working-directory: ./docs - run: pnpm install - - name: Run Astro check - working-directory: ./docs - run: pnpm check - - name: Run docs smoke checks - working-directory: ./docs - run: pnpm smoke - build: name: Validate Build runs-on: ubuntu-latest - needs: [format, smoke] + needs: [format] steps: - uses: actions/checkout@v6 - name: Setup Node.js diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index fb94ae1e1..a127dbad4 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -18,16 +18,24 @@ export default defineConfig({ social: [{ icon: 'github', label: 'GitHub', href: 'https://github.com/embucket/embucket' }], sidebar: [ { - label: 'Essentials', - autogenerate: { directory: 'essentials' }, + label: 'Getting Started', + autogenerate: { directory: 'getting-started' }, }, { - label: 'Guides', - autogenerate: { directory: 'guides' }, + label: 'Deploy', + autogenerate: { directory: 'deploy' }, }, { - label: 'Development', - autogenerate: { directory: 'development' }, + label: 'Connect', + autogenerate: { directory: 'connect' }, + }, + { + label: 'Tutorials', + autogenerate: { directory: 'tutorials' }, + }, + { + label: 'Reference', + autogenerate: { directory: 'reference' }, }, ], customCss: ['./src/styles/global.css'], @@ -46,6 +54,6 @@ export default defineConfig({ plugins: [tailwindcss()], }, redirects: { - '/': '/essentials/quick-start/', + '/': '/getting-started/quick-start/', }, }); diff --git a/docs/package.json b/docs/package.json index 14b71bc0b..496210658 100644 --- a/docs/package.json +++ b/docs/package.json @@ -11,8 +11,7 @@ "format": "prettier --write .", "ncu": "ncu -u", "preview": "astro preview", - "smoke": "node ./scripts/validate-docs-smoke.mjs", - "start": "astro dev" +"start": "astro dev" }, "dependencies": { "@astrojs/starlight": "^0.36.2", diff --git a/docs/scripts/validate-docs-smoke.mjs b/docs/scripts/validate-docs-smoke.mjs deleted file mode 100644 index 7487a6f97..000000000 --- a/docs/scripts/validate-docs-smoke.mjs +++ /dev/null @@ -1,113 +0,0 @@ -import { readFileSync } from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const docsRoot = path.resolve(__dirname, '..'); -const repoRoot = path.resolve(docsRoot, '..'); - -function read(relativePath, base = repoRoot) { - return readFileSync(path.join(base, relativePath), 'utf8'); -} - -function assert(condition, message) { - if (!condition) { - throw new Error(message); - } -} - -function assertIncludes(content, expected, label) { - assert(content.includes(expected), `${label} is missing expected text: ${expected}`); -} - -const quickStart = read('docs/src/content/docs/essentials/quick-start.mdx'); -const snowflakeCli = read('docs/src/content/docs/guides/snowflake-cli.mdx'); -const troubleshooting = read('docs/src/content/docs/guides/troubleshooting.mdx'); -const runtimeModes = read('docs/src/content/docs/essentials/runtime-modes.mdx'); -const configuration = read('docs/src/content/docs/essentials/configuration.mdx'); -const supportMatrix = read('docs/src/content/docs/essentials/support-matrix.mdx'); -const dbtGuide = read('docs/src/content/docs/guides/dbt.mdx'); -const endToEndDbt = read('docs/src/content/docs/guides/end-to-end-dbt.mdx'); -const docsMaintenance = read('docs/src/content/docs/development/docs-maintenance.mdx'); -const awsLambda = read('docs/src/content/docs/guides/aws-lambda.mdx'); -const s3Tables = read('docs/src/content/docs/guides/s3-tables.mdx'); -const selfHosted = read('docs/src/content/docs/guides/self-hosted.mdx'); -const snowplow = read('docs/src/content/docs/guides/snowplow.mdx'); -const docsReadme = read('docs/README.md'); -const packageJson = JSON.parse(read('docs/package.json')); - -const highTrafficGuides = [ - 'docs/src/content/docs/essentials/quick-start.mdx', - 'docs/src/content/docs/essentials/runtime-modes.mdx', - 'docs/src/content/docs/guides/aws-lambda.mdx', - 'docs/src/content/docs/guides/dbt.mdx', - 'docs/src/content/docs/guides/end-to-end-dbt.mdx', - 'docs/src/content/docs/guides/self-hosted.mdx', - 'docs/src/content/docs/guides/snowplow.mdx', - 'docs/src/content/docs/guides/troubleshooting.mdx', -]; - -for (const guidePath of highTrafficGuides) { - const content = read(guidePath); - assert(/> Owner:/m.test(content), `${guidePath} must include an Owner block near the top`); - assert( - /> Last reviewed:/m.test(content), - `${guidePath} must include a Last reviewed block near the top`, - ); -} - -for (const [content, label] of [ - [quickStart, 'Quick Start'], - [snowflakeCli, 'Snowflake CLI guide'], - [troubleshooting, 'Troubleshooting guide'], -]) { - assertIncludes(content, 'embucket/embucket', label); - assertIncludes(content, '3000', label); - assertIncludes(content, 'embucket', label); -} - -assertIncludes(quickStart, 'http://127.0.0.1:3000/', 'Quick Start'); -assertIncludes(runtimeModes, 'private API Gateway example', 'Runtime modes'); -assertIncludes(configuration, 'METASTORE_CONFIG=./metastore.yaml', 'Configuration guide'); -assertIncludes(configuration, 'volumes: []', 'Configuration guide'); -assertIncludes(snowflakeCli, 'protocol = "http"', 'Snowflake CLI guide'); -assertIncludes(troubleshooting, 'protocol = "http"', 'Troubleshooting guide'); -assertIncludes(selfHosted, 'cargo build', 'Self-hosted guide'); -assertIncludes(selfHosted, 'target/debug/embucketd', 'Self-hosted guide'); -assertIncludes(selfHosted, 'snow connection test', 'Self-hosted guide'); -assertIncludes(selfHosted, 'evaluation and testing', 'Self-hosted guide'); -assertIncludes(selfHosted, 'METASTORE_CONFIG=./metastore.yaml', 'Self-hosted guide'); -assertIncludes(awsLambda, 'AWS::ApiGateway::RestApi', 'AWS Lambda guide'); -assertIncludes(awsLambda, 'AWS::EC2::VPCEndpoint', 'AWS Lambda guide'); -assertIncludes(snowplow, 'embucket-snowplow', 'Snowplow guide'); -assertIncludes(snowplow, 'dbt run', 'Snowplow guide'); -assertIncludes(snowplow, 'dbt show', 'Snowplow guide'); -assertIncludes(snowplow, 'compatibility workaround', 'Snowplow guide'); -assertIncludes(runtimeModes, '/guides/self-hosted/', 'Runtime modes'); -assertIncludes(runtimeModes, '/guides/snowplow/', 'Runtime modes'); -assertIncludes(supportMatrix, '/guides/self-hosted/', 'Support matrix'); -assertIncludes(supportMatrix, '/guides/snowplow/', 'Support matrix'); -assertIncludes(dbtGuide, '/guides/snowplow/', 'dbt guide'); -assertIncludes(endToEndDbt, '/guides/snowplow/', 'End-to-end dbt guide'); -assertIncludes(awsLambda, '/guides/snowplow/', 'AWS Lambda guide'); -assertIncludes(s3Tables, '/guides/snowplow/', 'S3 Tables guide'); -assertIncludes(docsMaintenance, 'Self-hosted local binary', 'Docs maintenance'); -assertIncludes(docsMaintenance, 'Snowplow web analytics', 'Docs maintenance'); - -const requiredCommands = [ - 'pnpm dev', - 'pnpm build', - 'pnpm preview', - 'pnpm astro', - 'pnpm format', - 'pnpm ncu', -]; -for (const command of requiredCommands) { - assertIncludes(docsReadme, command, 'docs/README.md'); -} - -for (const scriptName of ['astro', 'build', 'check', 'format', 'ncu', 'preview', 'smoke']) { - assert(scriptName in packageJson.scripts, `docs/package.json is missing script: ${scriptName}`); -} - -console.log('Docs smoke checks passed.'); diff --git a/docs/src/content/docs/essentials/create-volume-1.png b/docs/src/content/docs/deploy/create-volume-1.png similarity index 100% rename from docs/src/content/docs/essentials/create-volume-1.png rename to docs/src/content/docs/deploy/create-volume-1.png diff --git a/docs/src/content/docs/essentials/create-volume-2.png b/docs/src/content/docs/deploy/create-volume-2.png similarity index 100% rename from docs/src/content/docs/essentials/create-volume-2.png rename to docs/src/content/docs/deploy/create-volume-2.png diff --git a/docs/src/content/docs/guides/s3-tables-query.png b/docs/src/content/docs/deploy/s3-tables-query.png similarity index 100% rename from docs/src/content/docs/guides/s3-tables-query.png rename to docs/src/content/docs/deploy/s3-tables-query.png diff --git a/docs/src/content/docs/development/docs-maintenance.mdx b/docs/src/content/docs/development/docs-maintenance.mdx deleted file mode 100644 index 6cfcf6cba..000000000 --- a/docs/src/content/docs/development/docs-maintenance.mdx +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Docs maintenance -description: How to review docs and keep them current. -sidebar: - order: 1 ---- - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -This page documents the current release-gate process for docs changes. - -## Automated checks - -The repository already includes a docs CI workflow at `.github/workflows/docs-ci.yml`. - -On docs changes, it currently runs: - -- formatting checks -- `astro check` -- repo-safe docs smoke checks for repeated constants, guide metadata, and referenced local files -- a full docs build -- Vale linting - -The docs site also uses `starlight-links-validator` during the build. - -## What still needs manual smoke testing - -Before release, manually check the commands in the highest-traffic guides: - -- `Quick Start` -- `Runtime modes` -- `Self-hosted local binary` -- `AWS Lambda` -- `dbt` -- `End-to-end dbt workflow` -- `Snowplow web analytics` -- `Troubleshooting` - -Confirm at least these points: - -- the local quick start still starts `embucket/embucket` -- the self-hosted guide still uses `cargo build` and can start `target/debug/embucketd` -- the Snowflake command-line tool can still run a simple query against the documented path -- the Lambda guide still reflects the current deploy flow -- the dbt guide still matches the current `dbt-embucket` adapter contract -- the Snowplow guide still matches the current `embucket-snowplow` flow and uses `dbt run` plus `dbt show` as the success state - -The automated smoke checks stay intentionally static. They do not call AWS, run dbt against a live Lambda, or execute the Snowflake command-line tool against live infrastructure. - -## Owner and freshness convention - -Major guides should include a short block near the top with: - -- owner -- last reviewed date - -This repository now uses that pattern in the highest-traffic guides so readers can see which pages the team reviewed recently. - -## Release checklist - -Before a release or launch push, confirm: - -1. current runtime names are consistent -2. quick-start ports and URLs still match the implementation -3. support claims still match verified workflows -4. docs CI is green -5. the top guides have a recent review date -6. self-hosted and Snowplow guides still match the current source repositories and local commands diff --git a/docs/src/content/docs/development/tracing.mdx b/docs/src/content/docs/development/tracing.mdx deleted file mode 100644 index f0c425de4..000000000 --- a/docs/src/content/docs/development/tracing.mdx +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: Tracing and profiling -description: Learn how to use tracing and profiling features in Embucket for debugging and performance analysis. ---- - -## Tracing - -Embucket uses `tracing::instrument` to instrument code for tracing. You can use it in both development and production environments. For development, use `info`, `debug`, or `trace` levels. For production, use the `info` level. - -### Tracing span processor experimental async runtime - -Embucket uses `BatchSpanProcessor`, which uses a dedicated background thread for collecting and exporting spans. This processor works well in production. If a development environment hangs on startup, switch to the experimental async runtime instead. -Use this command-line argument: `--tracing-span-processor=batch-span-processor-experimental-async-runtime`. - -### Tracing span processor tuning - -You can tune BatchSpanProcessor with the following environment variables: - -- `OTEL_BSP_MAX_CONCURRENT_EXPORTS`: Max number of concurrent export threads. Use this when running with the command-line argument: - `--tracing-span-processor=batch-span-processor-experimental-async-runtime` -- `OTEL_BSP_SCHEDULE_DELAY`: Frequency for batch exports, in milliseconds. Higher values reduce "BatchSpanProcessor. ExportError" messages in logs when you don't use an OpenTelemetry Protocol (OTLP) collector. -- `OTEL_BSP_EXPORT_TIMEOUT`: Max time allowed to export data. -- `OTEL_BSP_MAX_EXPORT_BATCH_SIZE`: Max number of spans per single export. -- `OTEL_BSP_MAX_QUEUE_SIZE`: Max number of spans you can buffer. - -### Logging - -Logging provides the basic way to observe debug and tracing events. -`RUST_LOG=debug` works for most cases. For tracing, use `RUST_LOG=trace`. - -### OpenTelemetry with Jaeger - -Instrumented calls in Embucket produce tracing events and spans using the [OpenTelemetry](https://opentelemetry.io/) SDK. These events go via OpenTelemetry Protocol (OTLP) to port 4317, where the OpenTelemetry Collector listens. -The collector starts collecting data when you run the Docker container, which also serves a [Jaeger](https://www.jaegertracing.io/) dashboard at [http://localhost:16686/](http://localhost:16686). - -```bash -# Run docker container with Jaeger UI v2 -docker run --rm --name jaeger -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 5778:5778 -p 9411:9411 jaegertracing/jaeger:2.6.0 -``` - -### Run Embucket in tracing mode - -Use the `RUST_LOG` environment variable to define log levels and the `--tracing-level` argument to enable tracing with [Jaeger](https://www.jaegertracing.io/). -Both default log level and default tracing level use `info`. - -```bash -target/debug/embucketd --jwt-secret=test --tracing-level=trace -``` - -## Profiling - -If you need to profile the `embucketd` executable, you can use [Samply](https://github.com/mstange/samply/). -This guide includes Samply as one way to profile, presented here as an experiment. This solution works out of the box on macOS, Linux, and Windows. - -To start profiling, prepend `samply record` to the `embucketd` command invocation. Perform the actions you need to profile, then stop profiling to open a profile report in the browser. - -```bash -# install Samply -cargo install --locked samply - -# Profile debug build -cargo build && samply record RUST_LOG=debug target/debug/embucketd --jwt-secret=test - -``` diff --git a/docs/src/content/docs/essentials/architecture.mdx b/docs/src/content/docs/essentials/architecture.mdx deleted file mode 100644 index 9d2595e09..000000000 --- a/docs/src/content/docs/essentials/architecture.mdx +++ /dev/null @@ -1,113 +0,0 @@ ---- -title: Architecture -description: Understand how Embucket handles data, metadata, query execution, authentication, and runtime packaging today. -sidebar: - order: 3 ---- - -import { Aside } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Embucket exposes a Snowflake-compatible API over lakehouse data. The current implementation is easiest to reason about if you separate it into five layers: runtime, metadata, storage, query execution, and auth/session handling. - -Use this page when you want the current mental model before choosing a runtime, configuring metadata, or planning around query and session limits. - -## Runtime model - -The repo currently ships two runtime artifacts: - -- `embucketd` for local and self-hosted runs -- `embucket-lambda` for AWS Lambda deployments - -The Lambda runtime wraps the same Snowflake-compatible router used by `embucketd`, so the client-facing API shape stays aligned across both runtime modes. - -Read [Runtime modes](/essentials/runtime-modes/) for the operational tradeoffs. - -If you are still choosing a path, start there before deploying. - -## Metadata model - -Embucket does not present itself as a managed metadata service. Instead, it loads metadata from configuration or supported external catalogs. - -Current documented paths are: - -- YAML metastore config loaded from `METASTORE_CONFIG` -- AWS S3 Tables as the documented external catalog path -- external Iceberg table definitions described in the metastore YAML - -If you need to configure metadata now, start with [Configuration](/essentials/configuration/) or [AWS S3 table bucket](/guides/s3-tables/). - - - -## Storage model - -Data stays in your object storage. The current docs and code paths center on: - -- Apache Iceberg metadata -- Parquet data files -- AWS S3 or S3-compatible object storage flows - -For AWS-managed table metadata, use [AWS S3 table bucket](/guides/s3-tables/). - -For a minimal local path, use the checked-in metastore config flow instead of starting with external catalog setup. - -## Query model - -Embucket executes Snowflake-flavored SQL through Apache DataFusion. The important operational property today is that query execution is **single-node per request**. - -That means: - -- each node handles complete queries independently -- there is no documented distributed query plan across many nodes -- scaling is primarily a throughput and placement question, not a distributed execution story - -This is why the docs distinguish local evaluation from the current Lambda deployment path instead of treating them as interchangeable production footprints. - -Plan for that limit before treating Lambda as a high-concurrency production endpoint. - -## Auth and session model - -The current Snowflake-compatible HTTP surface exposes: - -- `/session/v1/login-request` -- `/session` -- `/queries/v1/query-request` -- `/queries/v1/abort-request` - -The local demo path uses configurable demo credentials, which default to `embucket` / `embucket`. - -After login: - -- Embucket issues a JWT token -- clients can pass it in `Authorization: Snowflake Token="..."` -- the runtime also uses a `session_id` cookie for session continuity - -Current implementation details worth knowing: - -- JWT token lifetime is 3 days -- session inactivity expiry is 60 seconds - -If you are debugging auth problems, continue to [Troubleshooting](/guides/troubleshooting/). - -## When to use local mode or Lambda - -The API shape stays aligned, but the operational envelope changes: - -- **local mode** is the fastest path for tests and evaluation -- **Lambda + Function URL** is a tested validation path, but not production-ready because the URL is publicly reachable -- **Lambda + dbt-embucket** is the verified and recommended client workflow -- **private API gateway in front of Lambda** is the safer production-facing ingress pattern - -Use [Quick Start](/essentials/quick-start/) for local evaluation and [End-to-end dbt workflow](/guides/end-to-end-dbt/) for the shortest complete Lambda path. - -## Related guides - -- [Quick Start](/essentials/quick-start/) -- [Support matrix](/essentials/support-matrix/) -- [AWS Lambda](/guides/aws-lambda/) -- [dbt](/guides/dbt/) diff --git a/docs/src/content/docs/essentials/configuration.mdx b/docs/src/content/docs/essentials/configuration.mdx deleted file mode 100644 index 3020dd148..000000000 --- a/docs/src/content/docs/essentials/configuration.mdx +++ /dev/null @@ -1,146 +0,0 @@ ---- -title: Configuration -description: Configure Embucket with the current source-backed flags, environment variables, and metastore settings. -sidebar: - order: 2 ---- - -import { Aside } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -This page covers the current configuration surface backed by the repository. It focuses on settings that are visible in `crates/embucketd/src/cli.rs`, plus the checked-in config examples in `config/`. - -## Configuration precedence - -The current local runtime supports these configuration sources, from highest to lowest precedence: - -1. command-line flags -2. environment variables -3. `.env` loaded at startup - - - -## Core runtime settings - -| Purpose | Flag | Environment variable | Default | -| --------------------- | ------------------------- | ----------------------- | ----------- | -| metastore config path | `--metastore-config` | `METASTORE_CONFIG` | unset | -| bind host | `--host` | `BUCKET_HOST` | `localhost` | -| bind port | `--port` | `BUCKET_PORT` | `3000` | -| result serialization | `--data-format` | `DATA_FORMAT` | `json` | -| parser dialect | `--sql-parser-dialect` | `SQL_PARSER_DIALECT` | `snowflake` | -| query concurrency | `--max-concurrency-level` | `MAX_CONCURRENCY_LEVEL` | `8` | -| query timeout | `--query-timeout-secs` | `QUERY_TIMEOUT_SECS` | `1200` | -| demo user | `--auth-demo-user` | `AUTH_DEMO_USER` | `embucket` | -| demo password | `--auth-demo-password` | `AUTH_DEMO_PASSWORD` | `embucket` | -| JWT signing secret | `--jwt-secret` | `JWT_SECRET` | unset | -| tracing level | `--tracing-level` | `TRACING_LEVEL` | `info` | -| service idle timeout | `--idle-timeout-seconds` | `IDLE_TIMEOUT_SECONDS` | `18000` | - -## Local `.env` example - -Start with a minimal local `.env` like this: - -```bash -METASTORE_CONFIG=./metastore.yaml -JWT_SECRET=secret -TRACING_LEVEL=debug -RUST_LOG=info -``` - -Use stronger secrets than the example when you share an environment with anyone else. - -## Metastore configuration - -Embucket currently expects metastore configuration through `METASTORE_CONFIG` or `--metastore-config`. - -For a minimal local file, start with: - -```yaml -volumes: [] -``` - -That keeps the local runtime usable for evaluation and testing while you build out a richer metastore definition. - -### Configure the table-bucket volume - -```yaml -volumes: - - ident: embucket - type: s3-tables - database: demo - credentials: - credential_type: access_key - aws-access-key-id: ACCESS_KEY - aws-secret-access-key: SECRET_ACCESS_KEY - arn: arn:aws:s3tables:us-east-2:123456789012:bucket/my-table-bucket -``` - -Use this path if you want the currently documented external catalog flow. See [AWS S3 table bucket](/guides/s3-tables/) for the end-to-end setup. - -### External Iceberg tables on S3 - -```yaml -volumes: - - ident: lakehouse - type: s3 - region: us-east-2 - bucket: YOUR_BUCKET_NAME - credentials: - credential_type: access_key - aws-access-key-id: YOUR_ACCESS_KEY - aws-secret-access-key: YOUR_SECRET_KEY - -databases: - - ident: demo - volume: lakehouse - -schemas: - - database: demo - schema: tpch_10 - -tables: - - database: demo - schema: tpch_10 - table: customer - metadata_location: s3://YOUR_BUCKET_NAME/tpch_10/customer/metadata/00001.metadata.json -``` - -This path is useful when you already have Iceberg metadata in object storage and want Embucket to expose it through the Snowflake-compatible API. - -## Runtime and tracing settings - -The current runtime also exposes memory, disk, AWS SDK, and object-store timeout controls. The most important ones for everyday operation are: - -- `MEM_POOL_TYPE` -- `MEM_POOL_SIZE_MB` -- `DISK_POOL_SIZE_MB` -- `AWS_SDK_CONNECT_TIMEOUT_SECS` -- `AWS_SDK_OPERATION_TIMEOUT_SECS` -- `OBJECT_STORE_TIMEOUT_SECS` -- `OBJECT_STORE_CONNECT_TIMEOUT_SECS` -- `TRACING_LEVEL` -- `RUST_LOG` - -For deeper tracing guidance, see [Tracing and profiling](/development/tracing/). - -## Current authentication defaults - -Local quick starts assume demo authentication: - -- username: `embucket` -- password: `embucket` - -Override them with `AUTH_DEMO_USER` and `AUTH_DEMO_PASSWORD` when you need different demo credentials. - -## Related guides - -- [Quick Start](/essentials/quick-start/) -- [Runtime modes](/essentials/runtime-modes/) -- [AWS Lambda](/guides/aws-lambda/) -- [Troubleshooting](/guides/troubleshooting/) diff --git a/docs/src/content/docs/essentials/quick-start.mdx b/docs/src/content/docs/essentials/quick-start.mdx deleted file mode 100644 index 8433879ff..000000000 --- a/docs/src/content/docs/essentials/quick-start.mdx +++ /dev/null @@ -1,126 +0,0 @@ ---- -title: Quick Start -description: Start Embucket locally, connect the Snowflake command-line tool, and run a first query in the current test and evaluation path. -sidebar: - order: 0 ---- - -import { Aside, Steps } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -This quick start is the fastest way to try Embucket locally. This is the current **test and evaluation** path, not the recommended production deployment path. - -By the end of this guide, you can: - -- start the current `embucket/embucket` container -- connect the Snowflake command-line tool to the local endpoint -- run a query successfully - -## Step 1: Start Embucket - -Run the local container: - -```bash -docker run --name embucket --rm -p 3000:3000 embucket/embucket -``` - -Expected startup output includes a line like: - -```text -{"timestamp":"2025-07-01T15:35:05.687807Z","level":"INFO","fields":{"message":"Listening on http://0.0.0.0:3000"},"target":"embucketd"} -``` - - - -## Step 2: Configure the Snowflake command-line tool - -Install the Snowflake command-line tool if you do not have it already: - -```bash -python -m pip install snowflake-cli -``` - -Find your Snowflake command-line tool config file: - -```bash -snow --info -``` - -Add this connection block: - -```toml -[connections.local] -host = "localhost" -region = "us-east-2" -port = 3000 -protocol = "http" -database = "embucket" -schema = "public" -warehouse = "em.wh" -account = "acc.local" -user = "embucket" -password = "embucket" -``` - -Check the connection: - -```bash -snow connection test -c local -``` - -Expected output: - -```text -+-----------------------------+ -| key | value | -|-----------------+-----------| -| Connection name | local | -| Status | OK | -| Host | localhost | -| Account | acc | -| User | embucket | -| Role | not set | -| Database | embucket | -| Warehouse | em.wh | -+-----------------------------+ -``` - -## Step 3: Run your first query - -```bash -snow sql -c local -q "select dateadd(day, -1, current_timestamp()) as yesterday;" -``` - -Expected output: - -```text -+----------------------------------+ -| yesterday | -|----------------------------------| -| 2025-01-02 03:04:05.040000+00:00 | -+----------------------------------+ -``` - -## Step 4: Inspect the HTTP surface - -Open `http://127.0.0.1:3000/` in your browser to inspect the current Swagger/OpenAPI surface served by `embucketd`. - - - -## Next steps - -- If you want to run the local binary from source instead of Docker, read [Self-hosted local binary](/guides/self-hosted/). -- If you want the current runtime choices, read [Runtime modes](/essentials/runtime-modes/). -- If you want serverless deployment, read [AWS Lambda](/guides/aws-lambda/). -- If you want the recommended client path, read [dbt](/guides/dbt/). -- If you want a fuller analytics example on the recommended path, read [Snowplow web analytics](/guides/snowplow/). -- If you want an external catalog, read [AWS S3 table bucket](/guides/s3-tables/). -- If something fails, read [Troubleshooting](/guides/troubleshooting/). diff --git a/docs/src/content/docs/essentials/runtime-modes.mdx b/docs/src/content/docs/essentials/runtime-modes.mdx deleted file mode 100644 index 1c4979614..000000000 --- a/docs/src/content/docs/essentials/runtime-modes.mdx +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: Runtime modes -description: Choose between local evaluation, AWS Lambda validation, and the current recommended production-facing path. -sidebar: - order: 1 ---- - -import { Aside } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Embucket currently has two runtime artifacts: `embucketd` and `embucket-lambda`. The right path depends on whether you are evaluating the product, validating a client, or planning a production-facing deployment. - -If you already know what you want to do, use this page as a chooser: - -- Local evaluation → [Quick Start](/essentials/quick-start/) -- Source-built local binary → [Self-hosted local binary](/guides/self-hosted/) -- Lambda API validation → [AWS Lambda](/guides/aws-lambda/) -- Recommended client workflow → [dbt](/guides/dbt/) -- Shortest complete walkthrough → [End-to-end dbt workflow](/guides/end-to-end-dbt/) -- Fuller analytics example → [Snowplow web analytics](/guides/snowplow/) - -## Choose your path - -| Path | Best for | Current status | Start here | -| ------------------------------------------------------- | ------------------------------------------------------- | ---------------------------------------------------------- | --------------------------------------- | -| local `embucketd` via Docker or source build | tests, evaluation, local debugging | current local path | [Quick Start](/essentials/quick-start/) | -| AWS Lambda + Function URL + Snowflake command-line tool | validating the API with the Snowflake command-line tool | tested, but not production-ready because the URL is public | [AWS Lambda](/guides/aws-lambda/) | -| AWS Lambda + `dbt-embucket` adapter | dbt projects and repeatable workflows | verified and recommended | [dbt](/guides/dbt/) | -| AWS Lambda + private API gateway | production-facing ingress pattern | recommended deployment shape | [AWS Lambda](/guides/aws-lambda/) | - -## If you want local evaluation - -Use local mode when you want the shortest feedback loop. - -- default host: `localhost` -- default port: `3000` -- default demo credentials: `embucket` / `embucket` -- current browser-visible HTTP surface: `http://127.0.0.1:3000/` - -This is the fastest path for tests and evaluation, but the docs do not currently treat it as the primary production deployment recommendation. - -Start with [Quick Start](/essentials/quick-start/). - -If you want to build the binary yourself, use [Self-hosted local binary](/guides/self-hosted/). That path is currently for evaluation and testing rather than long-lived production deployment. - -## If you want the current Lambda runtime - -Use Lambda when you want the current serverless runtime. - -- runtime artifact: `embucket-lambda` -- deployment toolchain: `cargo lambda` plus the repo Makefile -- current test path: Function URL plus the Snowflake command-line tool -- current recommended client path: `dbt-embucket` - - - -For the recommended client path on Lambda, continue to [dbt](/guides/dbt/). - -If you want a fuller example on top of that path, continue to [Snowplow web analytics](/guides/snowplow/). - -## If you are deploying for production-facing traffic - -For production-facing traffic, the current guidance is: - -1. deploy `embucket-lambda` with the Lambda guide -2. avoid exposing a public Function URL as your main ingress -3. put an API gateway layer in front of the Lambda -4. use the dbt adapter as the primary documented client path -5. verify the full client path with the end-to-end dbt workflow - -The AWS Lambda guide includes an anonymized private API Gateway example for this ingress pattern. - -## Related guides - -- [Quick Start](/essentials/quick-start/) -- [Support matrix](/essentials/support-matrix/) -- [AWS Lambda](/guides/aws-lambda/) -- [dbt](/guides/dbt/) -- [End-to-end dbt workflow](/guides/end-to-end-dbt/) -- [Self-hosted local binary](/guides/self-hosted/) -- [Snowplow web analytics](/guides/snowplow/) diff --git a/docs/src/content/docs/essentials/snowflake.mdx b/docs/src/content/docs/essentials/snowflake.mdx deleted file mode 100644 index 44c48503d..000000000 --- a/docs/src/content/docs/essentials/snowflake.mdx +++ /dev/null @@ -1,134 +0,0 @@ ---- -title: Snowflake compatibility -description: What Embucket currently aims to support, what is explicitly verified, and where it still differs from Snowflake. -sidebar: - order: 5 ---- - -import { Aside, Steps } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this page to understand which Snowflake-style workflows work today, which ones have caveats, and where Embucket still differs from Snowflake. - -For the verified client/runtime combinations, read [Support matrix](/essentials/support-matrix/). - -## Current compatibility summary - -Embucket currently presents: - -- a Snowflake v1 REST API surface -- Snowflake-flavored SQL on top of Apache DataFusion -- client workflows for the Snowflake command-line tool and dbt -- a verified dbt adapter path through AWS Lambda - -## Before you rely on a workflow - -| Workflow shape | What to do | -| ------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| You want a verified path | use [Support matrix](/essentials/support-matrix/) and prefer the dbt adapter path | -| You want local evaluation | use [Quick Start](/essentials/quick-start/) and [Snowflake command-line tool](/guides/snowflake-cli/) | -| You want to understand differences from Snowflake | use the limitation sections below before release | - -## API compatibility - -Embucket provides a Snowflake v1 REST API that works with Snowflake-style login, session, and query flows. - -The current docs explicitly cover: - -- local Snowflake command-line tool against `embucketd` -- dbt via the `dbt-embucket` adapter on AWS Lambda -- Snowflake command-line tool against a Lambda Function URL for testing - -Choose one of those paths first instead of assuming all Snowflake-compatible tools behave the same way. - -## How the query engine behaves - -Embucket uses [Apache DataFusion](https://datafusion.apache.org/) for query execution. The goal is Snowflake-friendly behavior, but the current implementation still differs from Snowflake in important places. - -## Current limitations - -### Execution model - -- **Single-node execution**: query processing can use only one node's memory and CPU capacity -- **No documented distributed parallelism**: the current runtime story is not a distributed query engine - -### Data type limitations - -- **VARIANT**: Embucket stores it as JSON-serialized `TEXT` -- **Numeric coercion** can differ from Snowflake -- **Timestamp behavior** differs from Snowflake's precision and timezone semantics -- **Collation and character set support** remains limited - -### Error handling - -- error messages do not match Snowflake exactly - -### String escaping - -Backslash escaping differs from Snowflake in some cases. - -#### Literal backslashes - -**Snowflake:** - -```sql -SELECT * FROM VALUES ('\\b'); --- Returns: \b -``` - -**Embucket:** - -```sql -SELECT * FROM VALUES ('\\\\b'); --- Returns: \b -``` - -#### Single trailing backslash - -**Snowflake:** - -```sql -SELECT * FROM VALUES ('\\'); --- Returns: \ -``` - -**Embucket:** - -```sql -SELECT * FROM VALUES ('\\'); --- Error: Unterminated string literal -``` - -## VARIANT data type support - -Embucket implements VARIANT as JSON-serialized `TEXT` on the storage layer. - - - -1. **Create a table** - - ```sql - create table t2 (c1 variant) as values (parse_json('{"k1":1}')); - ``` - -2. **Read it back** - - ```sql - select * from t2; - ``` - -3. **Inspect the stored Arrow type** - - ```sql - select arrow_typeof(c1) from t2; - ``` - - - -## Next steps - -- Read [Support matrix](/essentials/support-matrix/) for verified workflows. -- Read [dbt](/guides/dbt/) for the recommended client path. -- Read [Troubleshooting](/guides/troubleshooting/) when compatibility expectations and current behavior diverge. diff --git a/docs/src/content/docs/essentials/support-matrix.mdx b/docs/src/content/docs/essentials/support-matrix.mdx deleted file mode 100644 index dc81e1d1a..000000000 --- a/docs/src/content/docs/essentials/support-matrix.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Support matrix -description: The currently documented and verified runtime and client combinations for Embucket. -sidebar: - order: 4 ---- - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this matrix to choose a supported runtime and client combination before you deploy, check, or troubleshoot. - -## Runtime and client matrix - -| Runtime + client path | Status | Recommended use | Evidence | -| --------------------------------------------------------- | ------------------------------ | ----------------------------------------------------- | ----------------------------------------------------------------------------------- | -| local `embucketd` + Snowflake command-line tool | documented | tests, evaluation, and local debugging | current quick start, self-hosted guide, and Snowflake command-line tool guide | -| AWS Lambda + `dbt-embucket` adapter | verified | recommended client path | official adapter tests run `dbt debug` and `dbt run` against a real Embucket Lambda | -| AWS Lambda + Snowflake command-line tool via Function URL | tested | API validation, but not production-facing traffic | current Lambda README and user guidance | -| AWS Lambda + private REST API gateway | recommended deployment pattern | production-facing ingress in front of the Lambda | AWS Lambda guide with an anonymized private API Gateway example | -| AWS S3 Tables external catalog | documented and supported | current external catalog path described in these docs | current S3 Tables guide and README | - -## Not currently covered in this docs set - -The current docs should **not** claim the following as verified unless new evidence supports them: - -- serverless deployments on GCP or Azure Functions -- a non-Lambda dbt transport for `dbt-embucket` -- broad benchmark conclusions not stated in official benchmark materials -- a full migration path from `embucket-labs` unless the docs include a dedicated migration guide - -## Names used in this guide - -- Use `embucket/embucket` for the current container image in public docs. -- Use `embucket-lambda` for the AWS Lambda runtime artifact. -- Treat `embucket-labs` as legacy naming unless a guide explicitly explains otherwise. - -## Example workflows in this docs set - -- [End-to-end dbt workflow](/guides/end-to-end-dbt/) is the shortest complete Lambda + dbt path. -- [Snowplow web analytics](/guides/snowplow/) is a fuller example built on the documented Lambda + `dbt-embucket` path. -- [Self-hosted local binary](/guides/self-hosted/) covers source-build local evaluation and testing. - -## Related guides - -- [Runtime modes](/essentials/runtime-modes/) -- [AWS Lambda](/guides/aws-lambda/) -- [dbt](/guides/dbt/) -- [End-to-end dbt workflow](/guides/end-to-end-dbt/) -- [Self-hosted local binary](/guides/self-hosted/) -- [Snowplow web analytics](/guides/snowplow/) -- [Troubleshooting](/guides/troubleshooting/) diff --git a/docs/src/content/docs/essentials/quick-start-ui.png b/docs/src/content/docs/getting-started/quick-start-ui.png similarity index 100% rename from docs/src/content/docs/essentials/quick-start-ui.png rename to docs/src/content/docs/getting-started/quick-start-ui.png diff --git a/docs/src/content/docs/guides/aws-lambda.mdx b/docs/src/content/docs/guides/aws-lambda.mdx deleted file mode 100644 index 2fedb444b..000000000 --- a/docs/src/content/docs/guides/aws-lambda.mdx +++ /dev/null @@ -1,255 +0,0 @@ ---- -title: AWS Lambda -description: Build, deploy, verify, and operate the current Embucket Lambda runtime. -sidebar: - order: 0 ---- - -import { Aside } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this guide when you want the current serverless runtime. The runtime artifact is `embucket-lambda`, and the repo includes both build metadata and deployment helpers for it. - - - -## Prerequisites - -- Rust toolchain -- `cargo-lambda` -- AWS credentials with permission to deploy and invoke Lambda -- a metastore config under `config/` -- optional env file such as `config/.env.lambda` - -## Deployment modes - -The current repo supports two distinct Lambda access patterns: - -- **Function URL + Snowflake command-line tool** for validation and manual testing -- **direct Lambda invoke via `dbt-embucket`** for the recommended client path - -Treat them differently: - -- Function URL is convenient for quick validation -- direct Lambda invoke avoids exposing a public endpoint -- private API Gateway is the safer production-facing ingress shape - -## Runtime defaults and config surface - -Current deploy metadata in the repo sets: - -| Setting | Current default | -| ------------- | -------------------------- | -| binary name | `bootstrap` | -| memory | `3008` | -| timeout | `30` | -| tracing | `Active` | -| included file | your metastore config file | - -Current deploy-time environment defaults in `Cargo.toml` are: - -| Variable | Default | -| ------------------ | -------------------------------------- | -| `LOG_FORMAT` | `json` | -| `METASTORE_CONFIG` | path to your packaged metastore config | -| `TRACING_LEVEL` | `debug` | -| `RUST_LOG` | `info` | - -Optional deploy-time inputs exposed by the Makefile include: - -| Variable | Purpose | -| --------------------- | -------------------------------------------------------- | -| `FUNCTION_NAME` | override the Lambda function name | -| `ENV_FILE` | inject env vars from a file such as `config/.env.lambda` | -| `AWS_LAMBDA_ROLE_ARN` | supply an execution role when creating a new function | -| `WITH_OTEL_CONFIG` | include an OpenTelemetry collector config file | -| `FEATURES` | enable optional cargo features such as `streaming` | -| `LAYERS` | attach extra Lambda layers | - -## Step 1: Build and deploy - -From the repository root: - -```bash -make -C crates/embucket-lambda deploy -``` - -This path builds `embucket-lambda`, deploys the `bootstrap` binary, and creates a Function URL for the test path. - -If you want to supply an env file explicitly: - -```bash -ENV_FILE=config/.env.lambda make -C crates/embucket-lambda deploy -``` - -If you need to deploy a new function with an explicit execution role: - -```bash -AWS_LAMBDA_ROLE_ARN=arn:aws:iam::123456789012:role/embucket-lambda-role make -C crates/embucket-lambda deploy -``` - -## IAM and access model - -Two IAM concerns matter in the current deployment story: - -1. **deployer permissions** for the human or CI identity running `cargo lambda deploy` -2. **execution role permissions** for the Lambda function itself - -The repo shows this explicitly: - -- the Makefile accepts `AWS_LAMBDA_ROLE_ARN` when creating a new function -- `dbt-embucket` uses AWS credentials to invoke the Lambda directly by ARN -- the private API template creates API Gateway permission to invoke the function - -Plan for at least: - -- permission to deploy or update the Lambda -- permission to read logs for verification -- permission for dbt clients to invoke the target function when using the recommended adapter path - -## Step 2: Verify the deployment - -The crate Makefile includes a built-in check: - -```bash -make -C crates/embucket-lambda verify -``` - -That runs: - -```bash -snow sql -c lambda -q "SELECT 1 as test_column" -``` - -You can also tail logs: - -```bash -make -C crates/embucket-lambda logs -``` - -For HTTP-level validation, the crate README also shows a direct login request: - -```bash -curl -X POST https://.lambda-url.us-east-2.on.aws/session/v1/login-request \ - -H "Content-Type: application/json" \ - -d '{"data": {"ACCOUNT_NAME": "account", "LOGIN_NAME": "embucket", "PASSWORD": "embucket", "CLIENT_APP_ID": "test"}}' -``` - -## What to verify before calling the deployment healthy - -- the function deploys successfully -- logs are readable in CloudWatch -- `snow sql -c lambda -q "SELECT 1 as test_column"` succeeds if you are validating the Function URL path -- `dbt debug` succeeds if you are validating the recommended adapter path -- the deployed config points at the intended metastore file - -## Step 3: Check the Snowflake command-line tool path over Function URL - -The current repo supports this as a test path. - -1. deploy the Lambda -2. capture the Function URL -3. create a `snow` connection that points to that URL -4. run a simple query such as `SELECT 1` - - - -## Step 4: Recommended production-facing ingress - -For production-facing traffic, keep the Lambda private and put an API gateway layer in front of it. - -Use a private API Gateway shape like this with your own IDs and names: - -```yaml -Parameters: - LambdaFunctionName: - Type: String - Default: embucket-lambda - VpcId: - Type: AWS::EC2::VPC::Id - Default: vpc-xxxxxxxx - SubnetIds: - Type: List - Default: subnet-aaaaaaaa,subnet-bbbbbbbb - VpcCidr: - Type: String - Default: 10.0.0.0/16 - -Resources: - ExecuteApiVpcEndpoint: - Type: AWS::EC2::VPCEndpoint - PrivateApi: - Type: AWS::ApiGateway::RestApi - LambdaInvokePermission: - Type: AWS::Lambda::Permission -``` - -This pattern provisions: - -- a private API Gateway -- a VPC endpoint for `execute-api` -- a Lambda proxy integration -- a stage named `v1` - -If your platform standard uses a custom domain, place it on top of the gateway layer rather than exposing a raw public Function URL. - -## Check limits and observability before rollout - -Before you treat a Lambda deployment as rollout-ready, check: - -- CloudWatch logs for runtime verification -- optional OpenTelemetry collector configuration via `WITH_OTEL_CONFIG` -- a state-store-backed pattern in production-style demos -- timeout and memory settings that match your expected workload -- whether your chosen ingress path matches your client path - -## Rollback and redeploy guidance - -The repo does not ship a one-command rollback wrapper, so the safest current guidance is operationally simple: - -1. keep the previous env file and metastore config under version control -2. keep the last known-good deployment inputs -3. redeploy the function with the previous config if a change regresses behavior -4. re-run the same verification command you used before the change - -If you introduce API Gateway or other surrounding infrastructure, roll it back through the same IaC path that created it. - -## Step 5: Recommended client path - -After the Lambda is live, use the [dbt adapter](/guides/dbt/) for the primary documented workflow. - -If you want the shortest full walkthrough, follow [End-to-end dbt workflow](/guides/end-to-end-dbt/). - -If you want a fuller project example on the same runtime path, follow [Snowplow web analytics](/guides/snowplow/). - -## Cleanup - -If you used the Function URL test path and want to remove it: - -```bash -aws lambda delete-function-url-config --function-name embucket-lambda -``` - -Also clean up any surrounding resources you created for the deployment path you chose: - -- Lambda function -- CloudWatch log group -- API Gateway and VPC endpoint if you used the private ingress path -- extra telemetry layers or config files if you attached them - -Remove infrastructure with the same tool that created it so the Lambda, API, and network state do not drift apart. - -## Common failure modes - -- **deploy succeeds but queries fail**: check that `METASTORE_CONFIG` points to a real packaged file -- **dbt cannot connect**: check AWS credentials and `EMBUCKET_FUNCTION_ARN` -- **Snowflake command-line tool works but should not be public**: move to the private API Gateway path -- **timeouts or truncated responses**: review timeout, memory, and whether you need `streaming` -- **no useful traces**: verify `RUST_LOG`, `TRACING_LEVEL`, and any OTEL config you attached diff --git a/docs/src/content/docs/guides/dbt.mdx b/docs/src/content/docs/guides/dbt.mdx deleted file mode 100644 index 34fa27d02..000000000 --- a/docs/src/content/docs/guides/dbt.mdx +++ /dev/null @@ -1,190 +0,0 @@ ---- -title: dbt -description: Use the official dbt-embucket adapter with the AWS Lambda runtime. -sidebar: - order: 1 ---- - -import { Aside, Steps } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -This is the recommended client path in the current docs. - -The official adapter lives in the sibling repository `Embucket/dbt-embucket`. Its current configuration model is Lambda-based: the profile uses `type: embucket` and `function_arn` to reach a deployed Embucket Lambda. - -## What is currently verified - -The official adapter repository includes live integration tests that run `dbt debug` and `dbt run` against a real Embucket Lambda. - -## Prerequisites - -- a deployed Embucket Lambda -- the Lambda function ARN -- AWS credentials that can invoke the function -- Python and dbt installed locally - -## What this guide gives you - -This page gives you enough material for a first successful setup: - -- install the adapter -- create a minimal project -- configure `profiles.yml` -- run `dbt debug` -- run `dbt run` on one model - -## Step 1: Install the adapter - -```bash -python -m pip install dbt-embucket -``` - -## Step 2: Create a minimal dbt project - -Create a new working directory and project files: - -```bash -mkdir embucket-dbt-demo -cd embucket-dbt-demo -mkdir -p models -``` - -Create `dbt_project.yml`: - -```yaml -name: embucket_demo -version: 1.0.0 -config-version: 2 - -profile: embucket - -model-paths: ['models'] - -models: - embucket_demo: - +materialized: view -``` - -Create `models/hello_embucket.sql`: - -```sql -select 1 as id, 'hello embucket' as message -``` - -## Step 3: Add a profile - -Add a profile like this to `profiles.yml`: - -```yaml -embucket: - target: dev - outputs: - dev: - type: embucket - function_arn: "{{ env_var('EMBUCKET_FUNCTION_ARN') }}" - account: "{{ env_var('EMBUCKET_ACCOUNT', 'embucket') }}" - user: "{{ env_var('EMBUCKET_USER', 'embucket') }}" - password: "{{ env_var('EMBUCKET_PASSWORD', 'embucket') }}" - database: "{{ env_var('EMBUCKET_DATABASE', 'demo') }}" - schema: public - threads: 1 -``` - -Export the Lambda ARN before running dbt: - -```bash -export EMBUCKET_FUNCTION_ARN=arn:aws:lambda:us-east-2:123456789012:function:embucket-lambda -``` - -If you want to keep credentials out of the file, also export these as needed: - -```bash -export EMBUCKET_ACCOUNT=embucket -export EMBUCKET_USER=embucket -export EMBUCKET_PASSWORD=embucket -export EMBUCKET_DATABASE=demo -``` - -### Profile field reference - -| Field | Required | Meaning | -| -------------- | -------- | ------------------------------------------------ | -| `type` | yes | must be `embucket` | -| `function_arn` | yes | target Lambda ARN | -| `account` | yes | logical account identifier sent in login payload | -| `user` | yes | Embucket login name | -| `password` | yes | Embucket login password | -| `database` | yes | default database | -| `schema` | yes | default schema | -| `threads` | yes | dbt concurrency setting | - -## Step 4: Check the connection - -```bash -dbt debug -``` - -Expected success signal: - -```text -Connection test: [OK connection ok] -All checks passed! -``` - -## Step 5: Run a model - -```bash -dbt run -``` - -Expected result: - -- dbt builds the `hello_embucket` model -- the run finishes successfully -- the object is available in the configured database and schema - -## Step 6: Verify the result from a client - -If you also have the Snowflake command-line tool configured for the same environment, run: - -```bash -snow sql -c lambda -q "select * from demo.public.hello_embucket" -``` - -You should see one row with `id = 1` and `message = 'hello embucket'`. - - - -## Current caveats - -- the official adapter is currently documented as a Lambda transport, not a local-runtime transport -- the adapter explicitly marks Python models as unsupported -- the adapter uses AWS Lambda invoke semantics rather than a long-lived TCP connection -- use the support matrix when you need to distinguish verified paths from documented-only paths - -## Common setup problems - -- **`profiles.yml` not found**: make sure dbt can see your profiles directory -- **`EMBUCKET_FUNCTION_ARN` missing**: export it before `dbt debug` -- **AWS credentials missing**: the adapter needs AWS credentials that can invoke the Lambda -- **auth failures**: check `EMBUCKET_USER` and `EMBUCKET_PASSWORD` -- **runs succeed but expected data is missing**: check database, schema, and metastore configuration on the Lambda side - -## Next step - -If you want the full ordered path from deploy to verification, read [End-to-end dbt workflow](/guides/end-to-end-dbt/). - -If you want a larger analytics example on the same adapter path, read [Snowplow web analytics](/guides/snowplow/). - -## Related guides - -- [AWS Lambda](/guides/aws-lambda/) -- [End-to-end dbt workflow](/guides/end-to-end-dbt/) -- [Snowplow web analytics](/guides/snowplow/) -- [Support matrix](/essentials/support-matrix/) -- [Troubleshooting](/guides/troubleshooting/) diff --git a/docs/src/content/docs/guides/end-to-end-dbt.mdx b/docs/src/content/docs/guides/end-to-end-dbt.mdx deleted file mode 100644 index aa47a1ab6..000000000 --- a/docs/src/content/docs/guides/end-to-end-dbt.mdx +++ /dev/null @@ -1,170 +0,0 @@ ---- -title: End-to-end dbt workflow -description: Deploy Embucket on AWS Lambda, connect dbt-embucket, run a minimal model, and verify the result end to end. -sidebar: - order: 2 ---- - -import { Aside, Steps } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -This guide is the shortest full workflow on the docs site. It takes you from a deployed Embucket Lambda to a successful dbt run and a verified query result. - - - -## What you need - -- AWS credentials -- a deployable Embucket checkout -- `cargo-lambda` -- Python with dbt and `dbt-embucket` -- a metastore config file for your deployment - -## Step 1: Deploy Embucket Lambda - -From the repo root: - -```bash -make -C crates/embucket-lambda deploy-only -``` - -If you want to inject an env file during deploy: - -```bash -ENV_FILE=config/.env.lambda make -C crates/embucket-lambda deploy-only -``` - -Record the function ARN for the next step. - -## Step 2: Install dbt-embucket - -```bash -python -m pip install dbt-embucket -``` - -## Step 3: Create a minimal dbt project - -```bash -mkdir embucket-dbt-demo -cd embucket-dbt-demo -mkdir -p models -``` - -Create `dbt_project.yml`: - -```yaml -name: embucket_demo -version: 1.0.0 -config-version: 2 - -profile: embucket - -model-paths: ['models'] - -models: - embucket_demo: - +materialized: view -``` - -Create `models/hello_embucket.sql`: - -```sql -select 1 as id, 'hello embucket' as message -``` - -## Step 4: Configure dbt - -Export the runtime values: - -```bash -export EMBUCKET_FUNCTION_ARN=arn:aws:lambda:us-east-2:123456789012:function:embucket-lambda -export EMBUCKET_ACCOUNT=embucket -export EMBUCKET_USER=embucket -export EMBUCKET_PASSWORD=embucket -export EMBUCKET_DATABASE=demo -``` - -Create `~/.dbt/profiles.yml`: - -```yaml -embucket: - target: dev - outputs: - dev: - type: embucket - function_arn: "{{ env_var('EMBUCKET_FUNCTION_ARN') }}" - account: "{{ env_var('EMBUCKET_ACCOUNT', 'embucket') }}" - user: "{{ env_var('EMBUCKET_USER', 'embucket') }}" - password: "{{ env_var('EMBUCKET_PASSWORD', 'embucket') }}" - database: "{{ env_var('EMBUCKET_DATABASE', 'demo') }}" - schema: public - threads: 1 -``` - -## Step 5: Check the connection - -```bash -dbt debug -``` - -Look for: - -```text -Connection test: [OK connection ok] -All checks passed! -``` - -## Step 6: Run the model - -```bash -dbt run -``` - -The run should create a `hello_embucket` object in the configured target schema. - -## Step 7: Verify the result - -If you have a `snow` connection for the same deployed runtime, verify with: - -```bash -snow sql -c lambda -q "select * from demo.public.hello_embucket" -``` - -Expected result shape: - -```text -+----+-----------------+ -| id | message | -|----+-----------------| -| 1 | hello embucket | -+----+-----------------+ -``` - -## Step 8: Clean up - -When you finish, remove: - -- the demo dbt project directory -- any local exported env vars or shell profile changes -- the deployed Lambda if you created that runtime only for evaluation - -Use the same deployment path that created the Lambda to tear it down cleanly. - -## Common failure points - -- `dbt debug` fails because `EMBUCKET_FUNCTION_ARN` is missing -- AWS credentials exist locally but do not have permission to invoke the Lambda -- the Lambda points at the wrong metastore config -- the database or schema in the dbt profile does not match the target you expect - -## Where to go next - -- For deeper Lambda operations, read [AWS Lambda](/guides/aws-lambda/) -- For adapter details, read [dbt](/guides/dbt/) -- For a fuller analytics example on the same path, read [Snowplow web analytics](/guides/snowplow/) -- For known failure modes, read [Troubleshooting](/guides/troubleshooting/) diff --git a/docs/src/content/docs/guides/s3-tables.mdx b/docs/src/content/docs/guides/s3-tables.mdx deleted file mode 100644 index a6d6a5b92..000000000 --- a/docs/src/content/docs/guides/s3-tables.mdx +++ /dev/null @@ -1,241 +0,0 @@ ---- -title: AWS S3 table bucket -description: Configure Embucket to read catalog metadata from an AWS S3 table bucket. ---- - -import { Aside, Steps, Tabs, TabItem } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this guide when you want Embucket to read catalog metadata from AWS S3 Tables. In the current docs set, S3 table buckets are the documented external catalog path. - -## What you'll learn - -Follow this guide to: - -- Create an AWS S3 table bucket using the AWS command-line tool -- Configure Embucket with an S3 table bucket as an external catalog -- Explore schemas, tables, and existing data -- Create tables and load data using familiar SQL commands -- Verify table creation and query data through AWS Console - -## Prerequisites - -Before you begin, verify you have: - -- AWS command-line tool installed and configured with appropriate permissions -- Embucket instance running locally or in your environment -- Valid AWS credentials with S3 Tables service permissions - -S3 table buckets map naturally to the current Embucket external catalog flow: one bucket ARN maps to one configured database. - - - -## Create an S3 table bucket - - - -1. **Create the table bucket** - - Use the AWS command-line tool to create your S3 table bucket: - - ```bash - aws s3tables create-table-bucket --name my-table-bucket --region us-east-2 - ``` - - The command returns the bucket ARN: - - ```json - { - "arn": "arn:aws:s3tables:us-east-2:123456789012:bucket/my-table-bucket" - } - ``` - -2. **Record the bucket information** - - Save the following information for the next step: - - **Bucket name**: `my-table-bucket` - - **Region**: `us-east-2` - - **ARN**: The full ARN returned by the command - - - -## Configure an Embucket volume - -In Embucket, volumes store data and metadata. Define configuration statically in the configuration file. Specify the configuration file location with the `--metastore-config` flag or the `METASTORE_CONFIG` environment variable. - -Create a section in the config file `volumes` and add an entry for the S3 table bucket volume. - -```yaml -volumes: - - ident: embucket - type: s3-tables - database: demo - credentials: - credential_type: access_key - aws-access-key-id: ACCESS_KEY - aws-secret-access-key: SECRET_ACCESS_KEY - arn: arn:aws:s3tables:us-east-2:123456789012:bucket/my-table-bucket -``` - -Parameters: - -- `ident`: Volume identifier -- `type`: Volume type `s3-tables`. This remains the only supported type. -- `database`: Database name that maps to this volume -- `credentials`: AWS access credentials -- `arn`: Full S3 table bucket ARN - - - -## Explore schemas and tables - -Explore schemas and tables in the S3 table bucket using the Snowflake command-line tool or any Snowflake-compatible tool. Use [Snowflake command-line tool guide](/guides/snowflake-cli) for the information on how to connect to Embucket. - - - -1. **Connect to Embucket** - - Start a Snowflake command-line tool session: - - ```bash - snow sql -c local - ``` - -2. **Explore schemas and tables** - - Explore schemas and tables: - - ```sql - SHOW SCHEMAS IN demo; - ``` - - Output: - - ``` - +----------------------------------------------------------------------------+ - | created_on | name | kind | database_name | schema_name | - |------------+--------------------------+------+---------------+-------------| - | None | public | None | demo | None | - | None | public_derived | None | demo | None | - | None | public_scratch | None | demo | None | - | None | public_snowplow_manifest | None | demo | None | - | None | tpcds_10 | None | demo | None | - | None | tpcds_100 | None | demo | None | - | None | tpch_10 | None | demo | None | - | None | tpch_100 | None | demo | None | - | None | information_schema | None | demo | None | - +----------------------------------------------------------------------------+ - ``` - -3. **Explore tables** - - Explore tables: - - ```sql - SHOW TABLES IN demo.tpch_10; - ``` - - Output: - - ``` - +-------------------------------------------------------------+ - | created_on | name | kind | database_name | schema_name | - |------------+----------+-------+---------------+-------------| - | None | orders | TABLE | demo | tpch_10 | - | None | nation | TABLE | demo | tpch_10 | - | None | customer | TABLE | demo | tpch_10 | - | None | part | TABLE | demo | tpch_10 | - | None | lineitem | TABLE | demo | tpch_10 | - | None | partsupp | TABLE | demo | tpch_10 | - | None | region | TABLE | demo | tpch_10 | - | None | supplier | TABLE | demo | tpch_10 | - +-------------------------------------------------------------+ - ``` - -4. **Create a table with data** - - Create and populate a table in one command: - - ```sql - CREATE TABLE demo.public.users ( - id INT, - name VARCHAR(100), - email VARCHAR(100) - ) AS VALUES - (1, 'John Doe', 'john.doe@example.com'), - (2, 'Jane Doe', 'jane.doe@example.com'); - ``` - - Output: - - ``` - +-------+ - | count | - |-------| - | 2 | - +-------+ - ``` - -5. **Query the table** - - Verify you can read the data: - - ```sql - SELECT * FROM demo.public.users; - ``` - - Output: - - ``` - +----+----------+----------------------+ - | id | name | email | - |----|----------|----------------------| - | 1 | John Doe | john.doe@example.com | - | 2 | Jane Doe | jane.doe@example.com | - +----+----------+----------------------+ - ``` - - - -## Verify in AWS console - -Verify table creation and query your data directly through AWS services: - - - -1. **Open AWS Console** - - Navigate to the S3 Tables service in the AWS Console. - -2. **Locate your table bucket** - - Find the `my-table-bucket` you created earlier. - -3. **Browse tables** - - Inside the table bucket, you see: - - Database: `demo` - - Table: `users` - -4. **Query with Athena** - - Select the `users` table and choose "Query table with Athena." The SQL editor opens with your table ready for queries. - - - -![S3 table bucket query interface](s3-tables-query.png) - -## Next steps - -Now that you have S3 table buckets working with Embucket, consider: - -- **Integration**: Connect BI tools and data pipelines to your Embucket instance -- **Analytics example**: If you want a fuller dbt-based workflow on Lambda, read [Snowplow web analytics](/guides/snowplow/) diff --git a/docs/src/content/docs/guides/self-hosted.mdx b/docs/src/content/docs/guides/self-hosted.mdx deleted file mode 100644 index d1cff1ffe..000000000 --- a/docs/src/content/docs/guides/self-hosted.mdx +++ /dev/null @@ -1,160 +0,0 @@ ---- -title: Self-hosted local binary -description: Build and run embucketd from source for local evaluation and testing. -sidebar: - order: 2 ---- - -import { Aside } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this guide when you want to build `embucketd` from source and run it locally. This path is currently for **evaluation and testing**, not the primary production deployment recommendation. - - - -## What you can do - -- build the workspace from source with `cargo build` -- start `target/debug/embucketd` with local config -- connect the Snowflake command-line tool to the local deployment -- run a query successfully - -## Prerequisites - -- Rust toolchain -- Python 3.8 or newer -- Snowflake command-line tool -- a clone of `Embucket/embucket` - -## Step 1: Build the repo - -From the repository root: - -```bash -cargo build -``` - -The repo contributing guide uses `cargo build` as the standard source-build path. - -## Step 2: Create a minimal metastore file - -Create a local `metastore.yaml` file in the repo root: - -```yaml -volumes: [] -``` - -## Step 3: Create a local env file - -Create a local `.env` file: - -```bash -cat > .env <<'EOF' -METASTORE_CONFIG=./metastore.yaml -JWT_SECRET=secret -TRACING_LEVEL=debug -RUST_LOG=info -EOF -``` - -That file sets: - -- `METASTORE_CONFIG=./metastore.yaml` -- `JWT_SECRET=secret` -- `TRACING_LEVEL=debug` -- `RUST_LOG=info` - - - -## Step 4: Start the binary - -Run the local binary from the repo root: - -```bash -target/debug/embucketd -``` - -Expected startup signals: - -- the process stays running -- the service binds to `localhost:3000` -- you can open `http://127.0.0.1:3000/` in your browser - -## Step 5: Configure the Snowflake command-line tool - -Find your Snowflake command-line tool config file: - -```bash -snow --info -``` - -Add this connection block: - -```toml -[connections.embucket] -host = "localhost" -region = "us-east-2" -port = 3000 -protocol = "http" -database = "embucket" -schema = "public" -warehouse = "em.wh" -account = "acc.local" -user = "embucket" -password = "embucket" -``` - -## Step 6: Verify the local deployment - -Check the connection: - -```bash -snow connection test -c embucket -``` - -Run a query: - -```bash -snow sql -c embucket -q "SELECT 1 AS ok" -``` - -Expected output: - -```text -+----+ -| ok | -|----| -| 1 | -+----+ -``` - -## Current limits of this path - -- this is the current source-build local workflow for evaluation and testing -- this path is not the recommended production deployment path -- the starter metastore config is minimal and does not populate example tables by itself - -If you need an external catalog or richer metadata, continue to [AWS S3 table bucket](/guides/s3-tables/). - -## Troubleshooting - -- **binary not found**: make sure `cargo build` completed successfully and run from the repo root -- **auth issues**: the local defaults are `embucket` / `embucket` unless you changed `AUTH_DEMO_USER` or `AUTH_DEMO_PASSWORD` -- **missing config**: make sure `.env` points at `./metastore.yaml` -- **Snowflake command-line tool SSL/protocol errors**: set `protocol = "http"` - -## Related guides - -- [Quick Start](/essentials/quick-start/) -- [Snowflake command-line tool](/guides/snowflake-cli/) -- [Runtime modes](/essentials/runtime-modes/) -- [Troubleshooting](/guides/troubleshooting/) diff --git a/docs/src/content/docs/guides/snowflake-cli.mdx b/docs/src/content/docs/guides/snowflake-cli.mdx deleted file mode 100644 index 841b79add..000000000 --- a/docs/src/content/docs/guides/snowflake-cli.mdx +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: Snowflake command-line tool -description: Connect the Snowflake command-line tool to Embucket for the current local test and evaluation workflow. -sidebar: - order: 3 ---- - -import { Aside, Steps } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this guide when you want to run the Snowflake command-line tool against a local Embucket instance. This is the current **local test and evaluation** client flow. - - - -## Prerequisites - -- Python 3.8 or newer -- Snowflake command-line tool -- Docker or a source-built local Embucket binary -- a running local Embucket instance - -## Step 1: Start Embucket - -Use either local startup path: - -- the container path in [Quick Start](/essentials/quick-start/) -- the source-build path in [Self-hosted local binary](/guides/self-hosted/) - -For the Docker path, run: - -```bash -docker run --name embucket --rm -p 3000:3000 embucket/embucket -``` - -## Step 2: Configure the Snowflake command-line tool - -Find your config file: - -```bash -snow --info -``` - -Add this connection block: - -```toml -[connections.embucket] -host = "localhost" -region = "us-east-2" -port = 3000 -protocol = "http" -database = "embucket" -schema = "public" -warehouse = "em.wh" -account = "acc.local" -user = "embucket" -password = "embucket" -``` - -Check the connection: - -```bash -snow connection test -c embucket -``` - -## Step 3: Run a query - -```bash -snow sql -c embucket -q "SELECT 1 AS ok" -``` - -Expected output: - -```text -+----+ -| ok | -|----| -| 1 | -+----+ -``` - -## Step 4: Inspect the API surface - -Open `http://127.0.0.1:3000/` to inspect the current Swagger/OpenAPI surface served by `embucketd`. - -## Troubleshooting - -- **Protocol errors:** set `protocol = "http"` in the connection block. -- **Authentication failures:** use `embucket` / `embucket` unless you changed `AUTH_DEMO_USER` or `AUTH_DEMO_PASSWORD`. -- **Connection refused:** make sure the container is running and publishing port `3000`. -- **No data or schemas:** load a metastore config or follow the [AWS S3 table bucket](/guides/s3-tables/) guide. - -## Next steps - -- For the recommended client path, read [dbt](/guides/dbt/). -- To run the local binary from source, read [Self-hosted local binary](/guides/self-hosted/). -- For AWS deployment, read [AWS Lambda](/guides/aws-lambda/). -- For runtime tradeoffs, read [Runtime modes](/essentials/runtime-modes/). diff --git a/docs/src/content/docs/guides/snowplow.mdx b/docs/src/content/docs/guides/snowplow.mdx deleted file mode 100644 index f6d3e9b30..000000000 --- a/docs/src/content/docs/guides/snowplow.mdx +++ /dev/null @@ -1,161 +0,0 @@ ---- -title: Snowplow web analytics -description: Run the embucket-snowplow example on the documented AWS Lambda plus dbt path. -sidebar: - order: 5 ---- - -import { Aside } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this guide when you want a fuller analytics example on top of the documented AWS Lambda + `dbt-embucket` workflow. This guide follows the public `Embucket/embucket-snowplow` repository. - - - -## What success looks like - -By the end of this guide, you should be able to: - -- deploy the Snowplow example runtime path on Lambda -- run `dbt run` successfully -- inspect derived results with `dbt show` - -## Prerequisites - -- AWS credentials -- an AWS S3 Table Bucket ARN -- `uv` or a Python environment that can run the example project -- Git - -## Step 1: Clone the example repo - -```bash -git clone https://github.com/Embucket/embucket-snowplow.git -cd embucket-snowplow -``` - -This example runs without a Snowflake account. - -## Step 2: Set deploy-time values - -Set the stack name and S3 Table Bucket ARN: - -```bash -STACK_NAME="embucket-demo-$(whoami)-$(date +%s)" -BUCKET_ARN="arn:aws:s3tables:us-east-2:YOUR_ACCOUNT:bucket/YOUR_BUCKET" -``` - -Use your own deployment values here. Database, schema, credentials, and the Lambda ARN are all deployment-defined in this flow. - -## Step 3: Deploy the Lambda stack - -```bash -aws cloudformation deploy \ - --template-file deploy/embucket-lambda.cfn.yaml \ - --stack-name "$STACK_NAME" \ - --capabilities CAPABILITY_NAMED_IAM \ - --parameter-overrides S3TableBucketArn="$BUCKET_ARN" -``` - -Capture the Lambda ARN after deploy: - -```bash -LAMBDA_ARN=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ - --query 'Stacks[0].Outputs[?OutputKey==`LambdaFunctionArn`].OutputValue' \ - --output text) -echo "$LAMBDA_ARN" -``` - -## Step 4: Install dependencies - -```bash -uv sync -``` - -## Step 5: Configure the dbt profile - -Start from the example profile: - -```bash -cp profiles.yml.example profiles.yml -sed -i '' "s|YOUR_LAMBDA_ARN_HERE|$LAMBDA_ARN|" profiles.yml -``` - -Review the profile values before you run dbt. Credentials, database, and schema should match your deployment choices rather than a hardcoded docs default. - -## Step 6: Install dbt packages - -```bash -uv run dbt deps --profiles-dir . -``` - -## Step 7: Patch only if package checks require it - -Some dbt packages check `target.type == 'snowflake'` and do not recognize `embucket` yet. - -If your package has that kind of target-type check, run the example patch script: - -```bash -./scripts/patch_snowplow.sh -``` - -Treat this as a **compatibility workaround**. If your package already supports `embucket` or does not hardcode `snowflake` checks, prefer the unpatched path. - -## Step 8: Load the example source data - -```bash -uv run python scripts/load_data.py "$LAMBDA_ARN" -``` - -The example loader creates the required schemas and source tables for the Snowplow workflow. - -## Step 9: Run the pipeline - -```bash -uv run dbt seed --profiles-dir . -uv run dbt run --profiles-dir . -``` - -This is the primary success signal for the example workflow. - -## Step 10: Verify the results with `dbt show` - -Run `dbt show` against one of the derived relations produced by your deployment. - -For the official example layout, start with: - -```bash -uv run dbt show --profiles-dir . --inline "SELECT * FROM demo.atomic_derived.snowplow_web_page_views" --limit 10 -``` - -You can repeat that for other derived relations, for example: - -```bash -uv run dbt show --profiles-dir . --inline "SELECT * FROM demo.atomic_derived.snowplow_web_sessions" --limit 10 -uv run dbt show --profiles-dir . --inline "SELECT * FROM demo.atomic_derived.snowplow_web_users" --limit 10 -``` - -If your deployment uses different database or schema names, update the relation names accordingly. - -## Cleanup - -Delete the CloudFormation stack when you finish: - -```bash -aws cloudformation delete-stack --stack-name "$STACK_NAME" -``` - -The example repo notes that this step doesn't remove data from your S3 Table Bucket automatically. - -## Related guides - -- [AWS Lambda](/guides/aws-lambda/) -- [dbt](/guides/dbt/) -- [End-to-end dbt workflow](/guides/end-to-end-dbt/) -- [Troubleshooting](/guides/troubleshooting/) diff --git a/docs/src/content/docs/guides/troubleshooting.mdx b/docs/src/content/docs/guides/troubleshooting.mdx deleted file mode 100644 index 8369aac63..000000000 --- a/docs/src/content/docs/guides/troubleshooting.mdx +++ /dev/null @@ -1,139 +0,0 @@ ---- -title: Troubleshooting -description: Fix the most common runtime, client, auth, and deployment mistakes in the current Embucket docs. -sidebar: - order: 4 ---- - -import { Aside } from '@astrojs/starlight/components'; - -> Owner: Embucket maintainers -> Last reviewed: 2026-04-07 - -Use this page when the documented path doesn't behave as expected. - -## Local startup succeeds, but the browser step doesn't work - -Use port `3000` for the current local runtime. - -- start command: `docker run --name embucket --rm -p 3000:3000 embucket/embucket` -- current browser-visible HTTP surface: `http://127.0.0.1:3000/` - -If you expected a separate default UI on port `8080`, treat that as stale guidance. - -If you started the local runtime from source instead of Docker, follow [Self-hosted local binary](/guides/self-hosted/) to confirm the build, env file, and startup command. - -## Snowflake command-line tool fails with protocol or SSL errors - -Set the connection protocol explicitly: - -```toml -protocol = "http" -``` - -The local quick start and current Snowflake command-line tool guide both describe the local endpoint as plain HTTP on port `3000`. - -## Authentication fails locally - -Use these default demo credentials: - -- user: `embucket` -- password: `embucket` - -If you overrode `AUTH_DEMO_USER` or `AUTH_DEMO_PASSWORD`, update your client config to match. - -## Snowflake command-line tool connects, but data doesn't appear - -The runtime only exposes what you configure. - -Check that you have supplied: - -- `METASTORE_CONFIG=./metastore.yaml`, or -- a metastore config path with actual volumes, databases, schemas, or tables - -If you want the documented external catalog path, follow [AWS S3 table bucket](/guides/s3-tables/). - -## Docs or scripts still say `embucket-labs` - -For current public docs, use: - -- container image: `embucket/embucket` -- Lambda artifact: `embucket-lambda` - -Treat `embucket-labs` references in current docs as legacy drift unless a page explicitly calls out migration or historical context. - -## Lambda works with the Snowflake command-line tool, but public access creates a problem - -That concern makes sense. The current docs treat Function URLs as a test path, not the production-ready ingress path. - -Use a private API gateway layer in front of the Lambda instead. The [AWS Lambda](/guides/aws-lambda/) guide includes an anonymized private API Gateway example you can adapt. - -## Lambda deploy succeeded, but the runtime still fails - -Check these in order: - -- the packaged config file exists at the path referenced by `METASTORE_CONFIG` -- the function has the intended memory and timeout settings -- the log group contains the startup and query errors you expect to see -- your ingress path matches your client path: Function URL for Snowflake command-line tool testing, direct invoke for dbt - -If you recently changed env vars or metastore config, redeploy with the last known-good inputs and check again. - -## dbt can't connect - -Start with these checks: - -- install `dbt-embucket` -- set `EMBUCKET_FUNCTION_ARN` -- your AWS credentials can invoke the Lambda -- your profile has `type: embucket` - -Then run: - -```bash -dbt debug -``` - -If `dbt debug` still fails, also check: - -- your local AWS credential chain can invoke Lambda -- the ARN region is correct -- deploy the target Lambda and confirm the function responds -- the Lambda-side demo credentials still match the values in your profile - -## dbt runs, but the model doesn't appear - -Check the exact target database and schema from your profile. - -For the minimal example in the docs, verify with: - -```bash -snow sql -c lambda -q "select * from demo.public.hello_embucket" -``` - -If the object goes missing, check whether the Lambda-side metastore and the dbt target schema point at the same location. - -If you follow the larger example workflow, continue with [Snowplow web analytics](/guides/snowplow/) and verify the derived tables with `dbt show`. - -## Snowplow package checks fail on `target.type` - -Some dbt packages assume `target.type == 'snowflake'` and don't recognize `embucket` yet. - -If you follow the official Snowplow example, use the patch step documented in [Snowplow web analytics](/guides/snowplow/). Treat that as a compatibility workaround, not as a default step for every dbt workload. - -## Sessions or auth feel short-lived - -The current implementation uses: - -- JWT tokens with a 3-day lifetime -- session inactivity expiry of 60 seconds - -If a long-running interactive workflow feels fragile, prefer an automated client path such as dbt instead of treating the local demo path as a long-lived session environment. - -## Still stuck? - -Work from the support matrix outward: - -1. confirm you are on a documented runtime/client pair in [Support matrix](/essentials/support-matrix/) -2. check the runtime with [Quick Start](/essentials/quick-start/) or [AWS Lambda](/guides/aws-lambda/) -3. switch to the recommended client path in [dbt](/guides/dbt/) if you need a verified workflow diff --git a/docs/src/content/docs/essentials/architecture.png b/docs/src/content/docs/reference/architecture.png similarity index 100% rename from docs/src/content/docs/essentials/architecture.png rename to docs/src/content/docs/reference/architecture.png diff --git a/docs/superpowers/plans/2026-04-08-docs-restructure.md b/docs/superpowers/plans/2026-04-08-docs-restructure.md new file mode 100644 index 000000000..96a31190a --- /dev/null +++ b/docs/superpowers/plans/2026-04-08-docs-restructure.md @@ -0,0 +1,938 @@ +# Docs Restructure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Restructure Embucket public docs from a confusing repo-centric layout into a clean product-docs structure: Getting Started, Deploy, Connect, Tutorials, Reference. + +**Architecture:** Delete 7 old pages and 2 old directories; create 5 new directories with 9 pages. Update astro sidebar config, CI workflow, and package.json. Each page is written by a docs-writer agent, reviewed by a docs-reviewer agent, then stranger-audited at the end. README.md gets a full rewrite. + +**Tech Stack:** Astro Starlight, MDX, Vale linter (Google + write-good styles), pnpm + +**Spec:** `docs/superpowers/specs/2026-04-08-docs-restructure-design.md` + +--- + +### Task 1: Scaffold directories and update config + +**Files:** +- Modify: `docs/astro.config.mjs` +- Modify: `docs/package.json` +- Modify: `.github/workflows/docs-ci.yml` +- Delete: `docs/scripts/validate-docs-smoke.mjs` +- Delete: `docs/src/content/docs/essentials/` (entire directory) +- Delete: `docs/src/content/docs/guides/` (entire directory) +- Delete: `docs/src/content/docs/development/` (entire directory) +- Create: `docs/src/content/docs/getting-started/` +- Create: `docs/src/content/docs/deploy/` +- Create: `docs/src/content/docs/connect/` +- Create: `docs/src/content/docs/tutorials/` +- Create: `docs/src/content/docs/reference/` + +- [ ] **Step 1: Create new directories** + +```bash +mkdir -p docs/src/content/docs/getting-started +mkdir -p docs/src/content/docs/deploy +mkdir -p docs/src/content/docs/connect +mkdir -p docs/src/content/docs/tutorials +mkdir -p docs/src/content/docs/reference +``` + +- [ ] **Step 2: Copy images to new locations** + +Images referenced by pages need to move with them: + +```bash +# Architecture diagram stays with architecture page +cp docs/src/content/docs/essentials/architecture.png docs/src/content/docs/reference/architecture.png +# Quick start UI screenshot +cp docs/src/content/docs/essentials/quick-start-ui.png docs/src/content/docs/getting-started/quick-start-ui.png +# S3 tables query screenshot goes to deploy (absorbed into Lambda page) +cp docs/src/content/docs/guides/s3-tables-query.png docs/src/content/docs/deploy/s3-tables-query.png +# Copy any create-volume images +cp docs/src/content/docs/essentials/create-volume-*.png docs/src/content/docs/deploy/ 2>/dev/null || true +``` + +- [ ] **Step 3: Delete old directories and files** + +```bash +rm -rf docs/src/content/docs/essentials +rm -rf docs/src/content/docs/guides +rm -rf docs/src/content/docs/development +rm docs/scripts/validate-docs-smoke.mjs +``` + +- [ ] **Step 4: Update astro.config.mjs sidebar** + +Replace the sidebar config in `docs/astro.config.mjs`: + +```javascript +sidebar: [ + { + label: 'Getting Started', + autogenerate: { directory: 'getting-started' }, + }, + { + label: 'Deploy', + autogenerate: { directory: 'deploy' }, + }, + { + label: 'Connect', + autogenerate: { directory: 'connect' }, + }, + { + label: 'Tutorials', + autogenerate: { directory: 'tutorials' }, + }, + { + label: 'Reference', + autogenerate: { directory: 'reference' }, + }, +], +``` + +Also update the root redirect: + +```javascript +redirects: { + '/': '/getting-started/quick-start/', +}, +``` + +- [ ] **Step 5: Remove smoke script from package.json** + +Remove the `"smoke"` entry from `scripts` in `docs/package.json`: + +```json +"smoke": "node ./scripts/validate-docs-smoke.mjs" +``` + +Delete that line. + +- [ ] **Step 6: Update CI workflow** + +In `.github/workflows/docs-ci.yml`: + +1. Delete the entire `smoke` job (lines 54-89) +2. In the `build` job, change `needs: [format, smoke]` to `needs: [format]` + +- [ ] **Step 7: Verify scaffold builds** + +```bash +cd docs && pnpm install && pnpm check +``` + +Expected: passes (no content files yet, but config is valid) + +- [ ] **Step 8: Commit scaffold** + +```bash +git add -A docs/ .github/workflows/docs-ci.yml +git commit -m "docs: scaffold new directory structure, remove smoke checks" +``` + +--- + +### Task 2: Write Quick Start page + +**Files:** +- Create: `docs/src/content/docs/getting-started/quick-start.mdx` + +**Source material:** +- Old file: `essentials/quick-start.mdx` (read from git: `git show HEAD~1:docs/src/content/docs/essentials/quick-start.mdx`) +- Spec section: "Getting Started / Quick Start" + +- [ ] **Step 1: Draft page with docs-writer agent** + +Dispatch a docs-writer agent with this prompt: + +> Write a Starlight MDX page for `docs/src/content/docs/getting-started/quick-start.mdx`. +> +> **Frontmatter:** title "Quick Start", description "Try Embucket locally with Docker and run your first query.", sidebar order 0. +> +> **Purpose:** Try Embucket locally in 2 minutes before deploying to AWS. +> +> **Content to include (in order):** +> 1. One-line intro: "Try Embucket locally before deploying to AWS." +> 2. Step 1: Start Embucket — `docker run --name embucket --rm -p 3000:3000 embucket/embucket` with expected startup log showing `Listening on http://0.0.0.0:3000` +> 3. Step 2: Configure Snowflake CLI — install with `python -m pip install snowflake-cli`, find config with `snow --info`, add TOML connection block (host=localhost, region=us-east-2, port=3000, protocol=http, database=embucket, schema=public, warehouse=em.wh, account=acc.local, user=embucket, password=embucket). Test with `snow connection test -c local`. +> 4. Step 3: Run first query — `snow sql -c local -q "select dateadd(day, -1, current_timestamp()) as yesterday;"` with expected output. +> 5. Next steps section with exactly 3 links: Deploy to AWS Lambda (`/deploy/aws-lambda/`), Connect with Snowflake CLI (`/connect/snowflake-cli/`), Connect with dbt (`/connect/dbt/`) +> +> **Rules:** +> - No "Owner/Last reviewed" blockquotes +> - No web UI mention +> - No hedging language ("test and evaluation", "not production-ready") +> - No self-hosted binary references +> - Use Starlight `Steps` component for the numbered steps +> - Import `{ Aside, Steps }` from `@astrojs/starlight/components` +> - Write as clean product documentation +> - Vale linter uses Google style guide — use second person ("you"), active voice, present tense + +- [ ] **Step 2: Review page with docs-reviewer agent** + +Dispatch a docs-reviewer agent with this prompt: + +> Review `docs/src/content/docs/getting-started/quick-start.mdx` against these criteria: +> 1. No "Owner:", "Last reviewed:", or internal metadata visible +> 2. No repo-centric language ("the repo", "current docs", "currently documented") +> 3. No hedging ("test and evaluation path", "not production-ready", "the confusion around") +> 4. No web UI or HTTP surface mention +> 5. No self-hosted binary references +> 6. Next steps has exactly 3 links to /deploy/aws-lambda/, /connect/snowflake-cli/, /connect/dbt/ +> 7. Uses Starlight Steps component +> 8. All code blocks have language tags +> 9. Would pass Vale with Google style (second person, active voice, present tense) +> Report issues as a numbered list. If clean, say "PASS". + +- [ ] **Step 3: Fix any review findings** + +- [ ] **Step 4: Verify build** + +```bash +cd docs && pnpm build 2>&1 | head -50 +``` + +Expected: build succeeds, no broken links for this page + +- [ ] **Step 5: Commit** + +```bash +git add docs/src/content/docs/getting-started/quick-start.mdx +git commit -m "docs: add Quick Start page" +``` + +--- + +### Task 3: Write AWS Lambda deployment page + +**Files:** +- Create: `docs/src/content/docs/deploy/aws-lambda.mdx` + +**Source material:** +- Old files (from git): `guides/aws-lambda.mdx`, `guides/s3-tables.mdx`, `essentials/runtime-modes.mdx` +- `crates/embucket-lambda/Cargo.toml` — Lambda deploy metadata (memory 3008, timeout 30) +- `crates/embucket-lambda/Makefile` — deploy targets and variables +- `crates/state-store/README.md` — DynamoDB setup, env vars, table schema + +- [ ] **Step 1: Draft page with docs-writer agent** + +Dispatch a docs-writer agent with this prompt: + +> Write a Starlight MDX page for `docs/src/content/docs/deploy/aws-lambda.mdx`. +> +> **Frontmatter:** title "AWS Lambda", description "Deploy Embucket to AWS Lambda with S3 Tables storage.", sidebar order 0. +> +> **Purpose:** Single page covering the full path from zero to a running Embucket Lambda with S3 Tables. +> +> **Content sections (in order):** +> +> 1. **Intro** (2 sentences): Embucket runs as an AWS Lambda function using S3 Tables (Apache Iceberg) for storage. This guide covers the full deployment from creating your S3 table bucket through verifying a working query. +> +> 2. **Prerequisites**: Rust toolchain, `cargo-lambda` (`cargo install cargo-lambda`), AWS CLI installed and configured, AWS credentials with Lambda/S3Tables/IAM permissions. +> +> 3. **Create an S3 table bucket**: Use `aws s3tables create-table-bucket --name my-table-bucket --region us-east-2`. Show the JSON response with ARN. Tell user to save bucket name, region, and ARN. +> +> 4. **Configure the metastore**: Create a YAML config file at `config/metastore.yaml`: +> ```yaml +> volumes: +> - ident: embucket +> type: s3-tables +> database: demo +> credentials: +> credential_type: access_key +> aws-access-key-id: ACCESS_KEY +> aws-secret-access-key: SECRET_ACCESS_KEY +> arn: arn:aws:s3tables:us-east-2:123456789012:bucket/my-table-bucket +> ``` +> Add an Aside tip to replace placeholders with real values. +> +> 5. **Build and deploy**: From repo root: `make -C crates/embucket-lambda deploy`. Show env file variant: `ENV_FILE=config/.env.lambda make -C crates/embucket-lambda deploy`. Show role ARN variant: `AWS_LAMBDA_ROLE_ARN=arn:aws:iam::123456789012:role/embucket-lambda-role make -C crates/embucket-lambda deploy`. Mention the Makefile variables: `FUNCTION_NAME` (default: embucket-lambda), `ENV_FILE`, `AWS_LAMBDA_ROLE_ARN`, `FEATURES`, `LAYERS`, `WITH_OTEL_CONFIG`. +> +> 6. **Verify the deployment**: `make -C crates/embucket-lambda verify` (runs `snow sql -c lambda -q "SELECT 1 as test_column"`). Also `make -C crates/embucket-lambda logs` to tail CloudWatch. Show the direct curl login-request example for HTTP-level validation. +> +> 7. **IAM and access**: Two concerns: (a) deployer permissions — identity running `cargo lambda deploy`; (b) execution role — the Lambda function itself. Mention that `dbt-embucket` uses AWS credentials to invoke Lambda directly by ARN. Plan for: permission to deploy/update, permission to read logs, permission for dbt clients to invoke. +> +> 8. **Lambda sizing**: Default memory is 3008 MB, timeout is 30 seconds (set in Cargo.toml deploy metadata). 3008 MB is the standard Lambda maximum — increasing beyond that requires an AWS support ticket (up to 10 GB). Override memory/timeout through `cargo lambda deploy` flags or AWS Console. For larger datasets or complex queries, consider requesting a memory increase. Tracing is set to Active by default. +> +> 9. **Statestore (optional)**: For persistent query state across invocations, build with the state-store feature flag: `FEATURES=state-store-query make -C crates/embucket-lambda deploy`. Requires a DynamoDB table. Show the `aws dynamodb create-table` command creating table `embucket-statestore` with PK/SK keys and GSIs for query_id, request_id, session_id. Environment variables: `STATESTORE_TABLE_NAME` (default: embucket-statestore), `STATESTORE_DYNAMODB_ENDPOINT` (for local testing: http://localhost:8000), `AWS_DDB_ACCESS_KEY_ID`, `AWS_DDB_SECRET_ACCESS_KEY`, `AWS_DDB_SESSION_TOKEN` (optional, for temporary credentials). +> +> 10. **Production ingress**: For production traffic, keep the Lambda private and put an API Gateway in front. Show the CloudFormation skeleton: +> ```yaml +> Parameters: +> LambdaFunctionName: +> Type: String +> Default: embucket-lambda +> VpcId: +> Type: AWS::EC2::VPC::Id +> SubnetIds: +> Type: List +> VpcCidr: +> Type: String +> Default: 10.0.0.0/16 +> +> Resources: +> ExecuteApiVpcEndpoint: +> Type: AWS::EC2::VPCEndpoint +> PrivateApi: +> Type: AWS::ApiGateway::RestApi +> LambdaInvokePermission: +> Type: AWS::Lambda::Permission +> ``` +> Note: provisions a private API Gateway, VPC endpoint for execute-api, Lambda proxy integration, stage named v1. Caution Aside: a public Function URL is acceptable for testing but should not be used for production traffic. +> +> 11. **Rollback and redeploy**: Keep previous env file and metastore config in version control. Redeploy with previous config if a change causes regression. Re-run verification after rollback. +> +> 12. **Cleanup**: `aws lambda delete-function-url-config --function-name embucket-lambda` to remove Function URL. Also clean up Lambda function, CloudWatch log group, API Gateway and VPC endpoint if used, telemetry layers. +> +> 13. **Troubleshooting** (inline at bottom): Deploy succeeds but queries fail → check METASTORE_CONFIG points to real file. dbt cannot connect → check AWS credentials and EMBUCKET_FUNCTION_ARN. Timeouts or truncated responses → review timeout and memory settings. No useful traces → verify RUST_LOG, TRACING_LEVEL, OTEL config. +> +> **Rules:** +> - No "Owner/Last reviewed" blockquotes +> - No repo-centric language +> - Import `{ Aside, Steps }` from `@astrojs/starlight/components` +> - Use Steps component for the main deployment flow +> - Vale Google style: second person, active voice, present tense +> - This is the most important page in the docs — it must be thorough and clear + +- [ ] **Step 2: Review page with docs-reviewer agent** + +Dispatch a docs-reviewer agent: + +> Review `docs/src/content/docs/deploy/aws-lambda.mdx` against the spec at `docs/superpowers/specs/2026-04-08-docs-restructure-design.md`, section "Deploy / AWS Lambda". Check: +> 1. All 12 content sections present (prerequisites through troubleshooting) +> 2. S3 table bucket creation absorbed from old s3-tables guide +> 3. Statestore section with feature flag `-F state-store` and DynamoDB setup +> 4. Lambda sizing section mentioning 3008 MB default and AWS support ticket for >3008 +> 5. No "Owner/Last reviewed", no repo-centric language, no internal commentary +> 6. All code blocks have language tags +> 7. Asides used appropriately (caution for Function URL, tips for config) +> Report issues as a numbered list. If clean, say "PASS". + +- [ ] **Step 3: Fix any review findings** + +- [ ] **Step 4: Verify build** + +```bash +cd docs && pnpm build 2>&1 | head -50 +``` + +- [ ] **Step 5: Commit** + +```bash +git add docs/src/content/docs/deploy/aws-lambda.mdx +git commit -m "docs: add AWS Lambda deployment page" +``` + +--- + +### Task 4: Write Configuration page + +**Files:** +- Create: `docs/src/content/docs/deploy/configuration.mdx` + +**Source material:** +- Old file (from git): `essentials/configuration.mdx` +- `crates/embucketd/src/cli.rs` — for flag/env var reference +- `crates/embucket-lambda/Makefile` — Makefile variables +- `crates/state-store/README.md` — statestore env vars + +- [ ] **Step 1: Draft page with docs-writer agent** + +Dispatch a docs-writer agent: + +> Write a Starlight MDX page for `docs/src/content/docs/deploy/configuration.mdx`. +> +> **Frontmatter:** title "Configuration", description "Environment variables, flags, and metastore settings for Embucket.", sidebar order 1. +> +> **Purpose:** Reference for all configuration options. +> +> **Content sections:** +> +> 1. **Configuration precedence**: flags (highest) > environment variables > .env file (loaded at startup via dotenv). +> +> 2. **Core runtime settings** table: +> +> | Purpose | Flag | Environment variable | Default | +> |---|---|---|---| +> | metastore config path | `--metastore-config` | `METASTORE_CONFIG` | unset | +> | bind host | `--host` | `BUCKET_HOST` | `localhost` | +> | bind port | `--port` | `BUCKET_PORT` | `3000` | +> | result serialization | `--data-format` | `DATA_FORMAT` | `json` | +> | parser dialect | `--sql-parser-dialect` | `SQL_PARSER_DIALECT` | `snowflake` | +> | query concurrency | `--max-concurrency-level` | `MAX_CONCURRENCY_LEVEL` | `8` | +> | query timeout | `--query-timeout-secs` | `QUERY_TIMEOUT_SECS` | `1200` | +> | demo user | `--auth-demo-user` | `AUTH_DEMO_USER` | `embucket` | +> | demo password | `--auth-demo-password` | `AUTH_DEMO_PASSWORD` | `embucket` | +> | JWT signing secret | `--jwt-secret` | `JWT_SECRET` | unset | +> | tracing level | `--tracing-level` | `TRACING_LEVEL` | `info` | +> | service idle timeout | `--idle-timeout-seconds` | `IDLE_TIMEOUT_SECONDS` | `18000` | +> +> 3. **Metastore configuration**: Explain METASTORE_CONFIG pointing to a YAML file. Show minimal example (`volumes: []`). Show S3 Tables volume example: +> ```yaml +> volumes: +> - ident: embucket +> type: s3-tables +> database: demo +> credentials: +> credential_type: access_key +> aws-access-key-id: ACCESS_KEY +> aws-secret-access-key: SECRET_ACCESS_KEY +> arn: arn:aws:s3tables:us-east-2:123456789012:bucket/my-table-bucket +> ``` +> Show external Iceberg tables on S3 example: +> ```yaml +> volumes: +> - ident: lakehouse +> type: s3 +> region: us-east-2 +> bucket: YOUR_BUCKET_NAME +> credentials: +> credential_type: access_key +> aws-access-key-id: YOUR_ACCESS_KEY +> aws-secret-access-key: YOUR_SECRET_KEY +> +> databases: +> - ident: demo +> volume: lakehouse +> +> schemas: +> - database: demo +> schema: tpch_10 +> +> tables: +> - database: demo +> schema: tpch_10 +> table: customer +> metadata_location: s3://YOUR_BUCKET_NAME/tpch_10/customer/metadata/00001.metadata.json +> ``` +> +> 4. **Statestore settings** table: +> +> | Variable | Purpose | Default | +> |---|---|---| +> | `STATESTORE_TABLE_NAME` | DynamoDB table name | `embucket-statestore` | +> | `STATESTORE_DYNAMODB_ENDPOINT` | DynamoDB endpoint (local testing) | unset | +> | `AWS_DDB_ACCESS_KEY_ID` | DynamoDB access key | unset | +> | `AWS_DDB_SECRET_ACCESS_KEY` | DynamoDB secret key | unset | +> | `AWS_DDB_SESSION_TOKEN` | Temporary credentials token | unset | +> +> 5. **Lambda deploy-time variables** table: +> +> | Variable | Purpose | +> |---|---| +> | `FUNCTION_NAME` | Override Lambda function name (default: embucket-lambda) | +> | `ENV_FILE` | Env file path (default: config/.env.lambda) | +> | `AWS_LAMBDA_ROLE_ARN` | Execution role ARN for new functions | +> | `WITH_OTEL_CONFIG` | OpenTelemetry collector config file path | +> | `FEATURES` | Cargo features, comma-separated (e.g., `state-store-query`) | +> | `LAYERS` | Additional Lambda layer ARNs | +> +> 6. **Memory and performance tuning** table: +> +> | Variable | Purpose | Default | +> |---|---|---| +> | `MEM_POOL_TYPE` | Memory pool type | unset | +> | `MEM_POOL_SIZE_MB` | Memory pool size in MB | unset | +> | `DISK_POOL_SIZE_MB` | Disk pool size in MB | unset | +> | `AWS_SDK_CONNECT_TIMEOUT_SECS` | AWS SDK connection timeout | unset | +> | `AWS_SDK_OPERATION_TIMEOUT_SECS` | AWS SDK operation timeout | unset | +> | `OBJECT_STORE_TIMEOUT_SECS` | Object store operation timeout | 30 | +> | `OBJECT_STORE_CONNECT_TIMEOUT_SECS` | Object store connection timeout | 3 | +> +> 7. **Authentication defaults**: Demo credentials are `embucket`/`embucket`. Override with `AUTH_DEMO_USER` and `AUTH_DEMO_PASSWORD`. Use stronger credentials in shared environments. +> +> **Rules:** +> - No "Owner/Last reviewed" blockquotes +> - No "backed by the repository" or "visible in cli.rs" language +> - Import `{ Aside }` from `@astrojs/starlight/components` +> - Vale Google style + +- [ ] **Step 2: Review page with docs-reviewer agent** + +> Review `docs/src/content/docs/deploy/configuration.mdx`. Check: all 7 sections present, statestore vars included, Lambda Makefile vars included, no internal language, tables are complete and accurate. + +- [ ] **Step 3: Fix any review findings** + +- [ ] **Step 4: Verify build and commit** + +```bash +cd docs && pnpm build 2>&1 | head -50 +git add docs/src/content/docs/deploy/configuration.mdx +git commit -m "docs: add Configuration reference page" +``` + +--- + +### Task 5: Write Snowflake CLI page + +**Files:** +- Create: `docs/src/content/docs/connect/snowflake-cli.mdx` + +**Source material:** +- Old file (from git): `guides/snowflake-cli.mdx` + +- [ ] **Step 1: Draft page with docs-writer agent** + +Dispatch a docs-writer agent: + +> Write a Starlight MDX page for `docs/src/content/docs/connect/snowflake-cli.mdx`. +> +> **Frontmatter:** title "Snowflake CLI", description "Connect the Snowflake command-line tool to Embucket.", sidebar order 0. +> +> **Purpose:** Connect the standard Snowflake CLI to a running Embucket instance. +> +> **Content:** +> 1. Intro (one line): The standard Snowflake command-line tool works with Embucket through its Snowflake-compatible REST API. +> 2. Prerequisites: Python 3.8+, Snowflake CLI (`python -m pip install snowflake-cli`), a running Embucket instance (local via Docker or deployed Lambda) +> 3. Configure connection: Find config with `snow --info`. Show TOML block for local: `[connections.local]` with host=localhost, region=us-east-2, port=3000, protocol=http, database=embucket, schema=public, warehouse=em.wh, account=acc.local, user=embucket, password=embucket. Test with `snow connection test -c local`. +> 4. Run a query: `snow sql -c local -q "SELECT 1 AS ok"` with expected output. +> 5. Troubleshooting (inline): Protocol errors → set protocol=http. Auth failures → use embucket/embucket. Connection refused → check Docker is running on port 3000. No data → configure metastore or see S3 table bucket setup in deploy guide. +> +> Wherever a Function URL or Lambda deployment detail is mentioned, add an ` -8. ### Load example data +8. **Load example data** Load sample Snowplow event data into your S3 Table Bucket: @@ -126,7 +126,7 @@ Before you begin, make sure you have the following: uv run python scripts/load_data.py "$LAMBDA_ARN" ``` -9. ### Run the pipeline +9. **Run the pipeline** Seed reference data and run the dbt transformations: @@ -135,7 +135,7 @@ Before you begin, make sure you have the following: uv run dbt run --profiles-dir . ``` -10. ### Verify the results +10. **Verify the results** Query the derived tables to confirm the pipeline completed successfully. From da6b8b69150d8d2bf3a4ffbdfdf5052806dea8db Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Wed, 8 Apr 2026 15:19:15 -0700 Subject: [PATCH 20/25] docs: fix all Vale warnings (passive voice, wordy, heading case) --- .github/styles/config/vocabularies/embucket/accept.txt | 4 ++++ docs/src/content/docs/configuration/configuration.mdx | 6 +++--- docs/src/content/docs/connect/dbt.mdx | 4 ++-- docs/src/content/docs/deploy/aws-lambda.mdx | 10 +++++----- docs/src/content/docs/getting-started/quick-start.mdx | 4 ++-- docs/src/content/docs/reference/snowflake.mdx | 10 +++++----- docs/src/content/docs/reference/troubleshooting.mdx | 8 ++++---- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.github/styles/config/vocabularies/embucket/accept.txt b/.github/styles/config/vocabularies/embucket/accept.txt index 32100a76c..5570cd4d1 100644 --- a/.github/styles/config/vocabularies/embucket/accept.txt +++ b/.github/styles/config/vocabularies/embucket/accept.txt @@ -51,3 +51,7 @@ bootable APIs? ARNs? [Dd]eployers? +CLI +[Ss]tatestore +GitLab +[Oo]pen-source diff --git a/docs/src/content/docs/configuration/configuration.mdx b/docs/src/content/docs/configuration/configuration.mdx index 0a8764f96..8cf35cd73 100644 --- a/docs/src/content/docs/configuration/configuration.mdx +++ b/docs/src/content/docs/configuration/configuration.mdx @@ -38,7 +38,7 @@ When the same setting appears at more than one level, the higher-priority source ## Metastore configuration -Embucket supports two ways to configure the metastore: a YAML configuration file or environment variables. The YAML file supports multiple volumes and full schema/table definitions. Environment variables configure a single volume and are useful for simple deployments. +Embucket supports two ways to configure the metastore: a YAML configuration file or environment variables. The YAML file supports many volumes and full schema/table definitions. Environment variables configure a single volume and are useful for simple deployments. @@ -96,7 +96,7 @@ tables: -Set the following environment variables to configure a single volume without a YAML file. When `VOLUME_TYPE` is set, Embucket uses these variables instead of `METASTORE_CONFIG`. +Set the following environment variables to configure a single volume without a YAML file. When you set `VOLUME_TYPE`, Embucket uses these variables instead of `METASTORE_CONFIG`. | Variable | Purpose | Default | | ----------------- | ---------------------------------------------- | ---------- | @@ -136,7 +136,7 @@ export VOLUME_SECRET_KEY=YOUR_SECRET_KEY ## Statestore settings -When the `state-store-query` feature is enabled, Embucket persists query state in DynamoDB. See [AWS Lambda deployment](/deploy/aws-lambda/) for how to enable this feature. Configure the table name and connection with the following variables. +When you enable the `state-store-query` feature, Embucket persists query state in DynamoDB. See [AWS Lambda deployment](/deploy/aws-lambda/) for how to enable this feature. Configure the table name and connection with the following variables. | Environment variable | Default | | ---------------------------------- | --------------------- | diff --git a/docs/src/content/docs/connect/dbt.mdx b/docs/src/content/docs/connect/dbt.mdx index 8ddb9a5f4..679cb6e39 100644 --- a/docs/src/content/docs/connect/dbt.mdx +++ b/docs/src/content/docs/connect/dbt.mdx @@ -7,7 +7,7 @@ sidebar: import { Aside, Steps } from '@astrojs/starlight/components'; -[dbt](https://docs.getdbt.com/) is a SQL-first transformation tool for analytics engineering. The `dbt-embucket` adapter connects to Embucket by invoking the Lambda function directly through AWS APIs, so no public endpoint is required. +[dbt](https://docs.getdbt.com/) is a SQL-first transformation tool for analytics engineering. The `dbt-embucket` adapter connects to Embucket by invoking the Lambda function directly through AWS APIs, so no public endpoint is necessary. ## Prerequisites @@ -145,7 +145,7 @@ Before you begin, make sure you have the following: +----+----------------+ ``` - + diff --git a/docs/src/content/docs/deploy/aws-lambda.mdx b/docs/src/content/docs/deploy/aws-lambda.mdx index bb7fff5e8..731d72587 100644 --- a/docs/src/content/docs/deploy/aws-lambda.mdx +++ b/docs/src/content/docs/deploy/aws-lambda.mdx @@ -237,7 +237,7 @@ Attach policies that grant the Lambda function access to the services it uses. T } ``` -**DynamoDB permissions (when state store is enabled):** +**DynamoDB permissions (when you enable the state store):** ```json { @@ -279,9 +279,9 @@ Users who connect through dbt-embucket need `lambda:InvokeFunctionUrl` or `lambd ## Lambda sizing -The default configuration sets memory to 3008 MB and timeout to 30 seconds. Tracing is set to `Active` by default. +The default configuration sets memory to 3008 MB and timeout to 30 seconds. Tracing defaults to `Active`. -3008 MB is the standard Lambda maximum. To increase memory beyond this limit (up to 10 GB), submit an AWS support ticket. +3008 MB is the standard Lambda limit. To increase memory beyond this cap (up to 10 GB), submit an AWS support ticket. Override memory and timeout through the AWS Console or the `aws lambda update-function-configuration` command. @@ -377,8 +377,8 @@ After removing the function URL, delete the following resources if they are no l **Deploy succeeds but queries fail** -- Check that the `METASTORE_CONFIG` environment variable points to a valid `metastore.yaml` and that the credentials and ARN inside the file are correct. -**dbt cannot connect** -- Verify that the client's AWS credentials have `lambda:InvokeFunction` permission on the function ARN. Confirm that `EMBUCKET_FUNCTION_ARN` is set correctly in your dbt profile. +**dbt cannot connect** -- Verify that the client's AWS credentials have `lambda:InvokeFunction` permission on the function ARN. Confirm that your dbt profile sets `EMBUCKET_FUNCTION_ARN` correctly. **Timeouts on large queries** -- Review the Lambda timeout and memory settings. Increase the timeout with `cargo lambda deploy` flags or through the AWS Console. Request a memory increase through AWS Support if you need more than 3008 MB. -**No traces or logs** -- Verify that `RUST_LOG` and `TRACING_LEVEL` are set in your environment file. If you use OpenTelemetry, confirm that `WITH_OTEL_CONFIG` points to a valid collector config and that the OTEL exporter endpoint is reachable. +**No traces or logs** -- Verify that your environment file defines `RUST_LOG` and `TRACING_LEVEL`. If you use OpenTelemetry, confirm that `WITH_OTEL_CONFIG` points to a valid collector config and that the OTEL exporter endpoint is reachable. diff --git a/docs/src/content/docs/getting-started/quick-start.mdx b/docs/src/content/docs/getting-started/quick-start.mdx index 1f3d817e5..22f4208e1 100644 --- a/docs/src/content/docs/getting-started/quick-start.mdx +++ b/docs/src/content/docs/getting-started/quick-start.mdx @@ -33,7 +33,7 @@ Try Embucket locally before deploying to AWS. python -m pip install snowflake-cli ``` - Find your configuration file path: + Find your configuration path: ```bash snow --info @@ -86,7 +86,7 @@ Try Embucket locally before deploying to AWS. snow sql -c local -q "select dateadd(day, -1, current_timestamp()) as yesterday;" ``` - The query returns a result similar to the following: + The query returns a result like the following: ```text +-------------------------------+ diff --git a/docs/src/content/docs/reference/snowflake.mdx b/docs/src/content/docs/reference/snowflake.mdx index d260ae406..2ed276390 100644 --- a/docs/src/content/docs/reference/snowflake.mdx +++ b/docs/src/content/docs/reference/snowflake.mdx @@ -20,7 +20,7 @@ Key features: Embucket exposes a Snowflake v1 REST API that works with any Snowflake client. The API handles login, session management, and query execution through the same endpoints that Snowflake clients expect. Testing focuses primarily on snowflake-connector-python and tools that depend on it, including dbt, snowflake-cli, and Superset. -## SQL engine +## Query engine Embucket uses [Apache DataFusion](https://datafusion.apache.org/) as its query engine -- an Apache Arrow-native analytical query engine. The goal is full SQL dialect compatibility with Snowflake. Embucket supports a growing set of Snowflake SQL features. DataFusion also includes some built-in functions that Snowflake does not provide. @@ -67,7 +67,7 @@ Error messages do not match Snowflake's error format. Arrow and DataFusion gener ### Backslash escaping -Backslash escaping differs from Snowflake in several cases. +Backslash escaping differs from Snowflake in the following cases. **Literal backslashes:** @@ -123,7 +123,7 @@ Error: Unterminated string literal ## VARIANT data type -Embucket implements VARIANT as JSON-serialized `TEXT` on the storage layer. The following example demonstrates how VARIANT values are stored and retrieved. +Embucket stores VARIANT values as JSON-serialized `TEXT` on the storage layer. The following example demonstrates how to create, read, and inspect VARIANT data. @@ -200,13 +200,13 @@ AVG(A) | SYSTEM$TYPEOF(AVG(A)) ## Timestamp handling -Arrow represents timestamps as 64-bit integers with nanosecond precision. This precision is fixed and cannot change per value. Snowflake uses variable precision for timestamps. +Arrow represents timestamps as 64-bit integers with nanosecond precision. This precision stays constant and cannot change per value. Snowflake uses variable precision for timestamps. Timezone handling also differs. Snowflake stores a timezone offset per timestamp value. Embucket stores a timezone offset per column. Query results may vary depending on the session timezone setting. ## Error message format -Arrow and DataFusion generate error messages that do not match Snowflake's error format. If your application parses Snowflake error codes or messages, expect differences when running against Embucket. +Arrow and DataFusion generate error messages that do not match Snowflake's error format. If your app parses Snowflake error codes or messages, expect differences when running against Embucket. ## Development roadmap diff --git a/docs/src/content/docs/reference/troubleshooting.mdx b/docs/src/content/docs/reference/troubleshooting.mdx index 2d1727d17..c591c2831 100644 --- a/docs/src/content/docs/reference/troubleshooting.mdx +++ b/docs/src/content/docs/reference/troubleshooting.mdx @@ -69,7 +69,7 @@ If data still does not appear, confirm that the credentials and resource ARNs in When a Lambda deployment succeeds but queries fail at runtime, check the following items in order: 1. **Metastore config** -- Verify that `METASTORE_CONFIG` points to the packaged `metastore.yaml` file inside the Lambda deployment artifact. -2. **Memory and timeout** -- Confirm that the Lambda function has sufficient memory (default is 3008 MB) and an appropriate timeout (default is 30 seconds). +2. **Memory and timeout** -- Confirm that the Lambda function has enough memory (default is 3008 MB) and an appropriate timeout (default is 30 seconds). 3. **CloudWatch logs** -- Open the log group for your Lambda function and look for error messages that identify the root cause. 4. **Ingress path** -- Confirm that the ingress path matches the client path. Use a Function URL for the Snowflake CLI and direct Lambda invoke for dbt. @@ -90,8 +90,8 @@ If `dbt debug` fails, work through the following checklist: Run `dbt debug` after each change to isolate the issue. Also check the following: - The AWS credential chain resolves to the correct identity. -- The ARN region matches the region where the Lambda function is deployed. -- The Lambda function is deployed and reachable. +- The ARN region matches the region where you deployed the Lambda function. +- The Lambda function exists and responds to requests. - The demo credentials in your profile match the values configured on the Lambda function. ## dbt runs but model doesn't appear @@ -121,7 +121,7 @@ This is a compatibility workaround. The script modifies package files to accept ## Sessions feel short-lived -Embucket issues JWT tokens with a 3-day lifetime. However, sessions expire after 60 seconds of inactivity. This means interactive sessions may appear to disconnect if you pause between queries. +Embucket issues JWT tokens with a 3-day lifetime. Sessions expire after 60 seconds of inactivity, so interactive sessions may appear to disconnect if you pause between queries. For long-running workflows, use automated tools like dbt rather than interactive sessions. The dbt adapter handles authentication and session management automatically. From 4815c96bde2ab1bfc40de9c74cca7ce2560e185c Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Wed, 8 Apr 2026 15:24:34 -0700 Subject: [PATCH 21/25] docs: fix Vale config - raise min level to warning, disable Vale.Terms false positives in mdx --- .vale.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vale.ini b/.vale.ini index 09290082c..33e4bf16c 100644 --- a/.vale.ini +++ b/.vale.ini @@ -1,6 +1,6 @@ StylesPath = .github/styles -MinAlertLevel = suggestion +MinAlertLevel = warning Packages = Google, write-good, MDX @@ -15,3 +15,4 @@ mdx = md [*.mdx] CommentDelimiters = {/*, */} TokenIgnores = (import\s+[^;]+;) +Vale.Terms = NO From a1596aa358e12ff2450d128513045de8cf884e01 Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Wed, 8 Apr 2026 15:27:56 -0700 Subject: [PATCH 22/25] docs: fix Vale errors with inline suppression for URL slugs, revert vale config --- .vale.ini | 3 +-- docs/src/content/docs/connect/dbt.mdx | 3 +++ docs/src/content/docs/connect/snowflake-cli.mdx | 3 +++ docs/src/content/docs/reference/architecture.mdx | 3 +++ docs/src/content/docs/reference/snowflake.mdx | 4 ++++ docs/src/content/docs/reference/troubleshooting.mdx | 3 +++ docs/src/content/docs/tutorials/snowplow.mdx | 3 +++ 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.vale.ini b/.vale.ini index 33e4bf16c..09290082c 100644 --- a/.vale.ini +++ b/.vale.ini @@ -1,6 +1,6 @@ StylesPath = .github/styles -MinAlertLevel = warning +MinAlertLevel = suggestion Packages = Google, write-good, MDX @@ -15,4 +15,3 @@ mdx = md [*.mdx] CommentDelimiters = {/*, */} TokenIgnores = (import\s+[^;]+;) -Vale.Terms = NO diff --git a/docs/src/content/docs/connect/dbt.mdx b/docs/src/content/docs/connect/dbt.mdx index 679cb6e39..7c07b5971 100644 --- a/docs/src/content/docs/connect/dbt.mdx +++ b/docs/src/content/docs/connect/dbt.mdx @@ -17,9 +17,12 @@ Before you begin, make sure you have the following: - **AWS credentials** with permission to invoke the Lambda function. - **Python 3.8 or later** and **dbt** installed on your system. +{/* vale Vale.Terms = NO */} + +{/* vale Vale.Terms = YES */} ## Set up dbt with Embucket diff --git a/docs/src/content/docs/connect/snowflake-cli.mdx b/docs/src/content/docs/connect/snowflake-cli.mdx index 948107ef9..1e6ed5120 100644 --- a/docs/src/content/docs/connect/snowflake-cli.mdx +++ b/docs/src/content/docs/connect/snowflake-cli.mdx @@ -22,10 +22,13 @@ Before you begin, make sure you have the following: - **A running Embucket instance.** Start one locally with Docker or deploy to AWS Lambda. +{/* vale Vale.Terms = NO */} + +{/* vale Vale.Terms = YES */} ## Set up the connection diff --git a/docs/src/content/docs/reference/architecture.mdx b/docs/src/content/docs/reference/architecture.mdx index 3d7a913a4..fe48e89b2 100644 --- a/docs/src/content/docs/reference/architecture.mdx +++ b/docs/src/content/docs/reference/architecture.mdx @@ -15,9 +15,12 @@ Embucket exposes a Snowflake-compatible API over lakehouse data. The system sepa Embucket runs as an AWS Lambda function (`embucket-lambda`) for production deployments. A local binary is also available through Docker for development and testing. Both entry points share the same Snowflake-compatible API router. +{/* vale Vale.Terms = NO */} + +{/* vale Vale.Terms = YES */} ## Metadata diff --git a/docs/src/content/docs/reference/snowflake.mdx b/docs/src/content/docs/reference/snowflake.mdx index 2ed276390..71a9fecc1 100644 --- a/docs/src/content/docs/reference/snowflake.mdx +++ b/docs/src/content/docs/reference/snowflake.mdx @@ -14,11 +14,15 @@ Key features: - Snowflake v1 REST API - SQL dialect powered by Apache DataFusion - Compatible with snowflake-connector-python and tools that depend on it + {/* vale Vale.Terms = NO */} - Integration with dbt, snowflake-cli, and Apache Superset + {/* vale Vale.Terms = YES */} ## API compatibility +{/* vale Vale.Terms = NO */} Embucket exposes a Snowflake v1 REST API that works with any Snowflake client. The API handles login, session management, and query execution through the same endpoints that Snowflake clients expect. Testing focuses primarily on snowflake-connector-python and tools that depend on it, including dbt, snowflake-cli, and Superset. +{/* vale Vale.Terms = YES */} ## Query engine diff --git a/docs/src/content/docs/reference/troubleshooting.mdx b/docs/src/content/docs/reference/troubleshooting.mdx index c591c2831..cfc31c3ec 100644 --- a/docs/src/content/docs/reference/troubleshooting.mdx +++ b/docs/src/content/docs/reference/troubleshooting.mdx @@ -73,10 +73,13 @@ When a Lambda deployment succeeds but queries fail at runtime, check the followi 3. **CloudWatch logs** -- Open the log group for your Lambda function and look for error messages that identify the root cause. 4. **Ingress path** -- Confirm that the ingress path matches the client path. Use a Function URL for the Snowflake CLI and direct Lambda invoke for dbt. +{/* vale Vale.Terms = NO */} + +{/* vale Vale.Terms = YES */} ## dbt can't connect diff --git a/docs/src/content/docs/tutorials/snowplow.mdx b/docs/src/content/docs/tutorials/snowplow.mdx index da1fdba96..c520cf155 100644 --- a/docs/src/content/docs/tutorials/snowplow.mdx +++ b/docs/src/content/docs/tutorials/snowplow.mdx @@ -26,10 +26,13 @@ Before you begin, make sure you have the following: - `uv` or another Python environment manager - Git +{/* vale Vale.Terms = NO */} + +{/* vale Vale.Terms = YES */} ## Tutorial From a4958d60387c89ff3dc5fded3679551d1446f225 Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Wed, 8 Apr 2026 15:43:56 -0700 Subject: [PATCH 23/25] docs: fix all Vale suggestions (contractions, E-Prime, parens, acronyms) --- .../config/vocabularies/embucket/accept.txt | 3 + .../docs/configuration/configuration.mdx | 44 ++++++++------- docs/src/content/docs/connect/dbt.mdx | 24 ++++---- .../content/docs/connect/snowflake-cli.mdx | 18 ++++-- docs/src/content/docs/deploy/aws-lambda.mdx | 56 ++++++++++--------- .../docs/getting-started/quick-start.mdx | 2 +- .../content/docs/reference/architecture.mdx | 10 +++- docs/src/content/docs/reference/snowflake.mdx | 31 +++++----- .../docs/reference/troubleshooting.mdx | 27 ++++++--- docs/src/content/docs/tutorials/snowplow.mdx | 4 +- 10 files changed, 129 insertions(+), 90 deletions(-) diff --git a/.github/styles/config/vocabularies/embucket/accept.txt b/.github/styles/config/vocabularies/embucket/accept.txt index 5570cd4d1..8d93fa868 100644 --- a/.github/styles/config/vocabularies/embucket/accept.txt +++ b/.github/styles/config/vocabularies/embucket/accept.txt @@ -55,3 +55,6 @@ CLI [Ss]tatestore GitLab [Oo]pen-source +TLS +VPC +OTEL diff --git a/docs/src/content/docs/configuration/configuration.mdx b/docs/src/content/docs/configuration/configuration.mdx index 8cf35cd73..3955c7377 100644 --- a/docs/src/content/docs/configuration/configuration.mdx +++ b/docs/src/content/docs/configuration/configuration.mdx @@ -7,7 +7,7 @@ sidebar: import { Aside, Tabs, TabItem } from '@astrojs/starlight/components'; -This page covers every configuration option for Embucket, including CLI flags, environment variables, metastore YAML, and deploy-time settings. It does not cover connection setup for clients such as the Snowflake CLI or dbt. +This page covers every configuration option for Embucket, including CLI flags, environment variables, metastore YAML, and deploy-time settings. It doesn't cover connection setup for clients such as the Snowflake CLI or dbt. ## Configuration precedence @@ -38,7 +38,7 @@ When the same setting appears at more than one level, the higher-priority source ## Metastore configuration -Embucket supports two ways to configure the metastore: a YAML configuration file or environment variables. The YAML file supports many volumes and full schema/table definitions. Environment variables configure a single volume and are useful for simple deployments. +Embucket supports two ways to configure the metastore: a YAML configuration file or environment variables. The YAML file supports many volumes and full schema/table definitions. Environment variables configure a single volume and work well for simple deployments. @@ -104,21 +104,25 @@ Set the following environment variables to configure a single volume without a Y | `VOLUME_IDENT` | Volume identifier | `embucket` | | `VOLUME_DATABASE` | Database name to associate with the volume | unset | +{/* vale Google.Parens = NO */} + **For S3 Tables (`VOLUME_TYPE=s3tables`):** -| Variable | Purpose | -| -------------------------- | ------------------------------- | -| `VOLUME_ARN` | S3 Tables bucket ARN (required) | -| `VOLUME_ACCESS_KEY` | AWS access key ID | -| `VOLUME_SECRET_KEY` | AWS secret access key | -| `VOLUME_AWS_SESSION_TOKEN` | AWS session token (optional) | +| Variable | Purpose | Required | +| -------------------------- | --------------------- | -------- | +| `VOLUME_ARN` | S3 Tables bucket ARN | Yes | +| `VOLUME_ACCESS_KEY` | AWS access key ID | No | +| `VOLUME_SECRET_KEY` | AWS secret access key | No | +| `VOLUME_AWS_SESSION_TOKEN` | AWS session token | No | **For S3 (`VOLUME_TYPE=s3`):** -| Variable | Purpose | -| ------------------- | -------------------------------- | -| `VOLUME_ACCESS_KEY` | AWS access key ID (required) | -| `VOLUME_SECRET_KEY` | AWS secret access key (required) | +| Variable | Purpose | Required | +| ------------------- | --------------------- | -------- | +| `VOLUME_ACCESS_KEY` | AWS access key ID | Yes | +| `VOLUME_SECRET_KEY` | AWS secret access key | Yes | + +{/* vale Google.Parens = YES */} Example for S3 Tables: @@ -138,13 +142,13 @@ export VOLUME_SECRET_KEY=YOUR_SECRET_KEY When you enable the `state-store-query` feature, Embucket persists query state in DynamoDB. See [AWS Lambda deployment](/deploy/aws-lambda/) for how to enable this feature. Configure the table name and connection with the following variables. -| Environment variable | Default | -| ---------------------------------- | --------------------- | -| `STATESTORE_TABLE_NAME` | `embucket-statestore` | -| `STATESTORE_DYNAMODB_ENDPOINT` | unset | -| `AWS_DDB_ACCESS_KEY_ID` | unset | -| `AWS_DDB_SECRET_ACCESS_KEY` | unset | -| `AWS_DDB_SESSION_TOKEN` (optional) | unset | +| Environment variable | Default | +| ------------------------------ | --------------------- | +| `STATESTORE_TABLE_NAME` | `embucket-statestore` | +| `STATESTORE_DYNAMODB_ENDPOINT` | unset | +| `AWS_DDB_ACCESS_KEY_ID` | unset | +| `AWS_DDB_SECRET_ACCESS_KEY` | unset | +| `AWS_DDB_SESSION_TOKEN` | unset | ## Lambda deploy-time variables @@ -175,7 +179,7 @@ Use these variables to adjust memory pools and network timeouts for your workloa ## Authentication defaults -Embucket ships with demo credentials for local development. The default username and password are both `embucket`. Override them with `AUTH_DEMO_USER` and `AUTH_DEMO_PASSWORD` before you expose the service to any network beyond localhost. +Embucket ships with demo credentials for local development. The default username and password both equal `embucket`. Override them with `AUTH_DEMO_USER` and `AUTH_DEMO_PASSWORD` before you expose the service to any network beyond localhost. +{/* vale Google.Parens = YES */} + ## Storage Data stays in your object storage. Embucket reads and writes data through three components: @@ -45,7 +49,7 @@ Data stays in your object storage. Embucket reads and writes data through three ## Query execution -Embucket executes Snowflake-compatible SQL through Apache DataFusion. Query execution is single-node per request. Each invocation handles a complete query independently. The engine does not distribute queries across nodes. +Embucket executes Snowflake-compatible SQL through Apache DataFusion. Query execution runs single-node per request. Each invocation handles a complete query independently. The engine doesn't distribute queries across nodes. ## Authentication and sessions @@ -56,4 +60,4 @@ Embucket provides a Snowflake-compatible HTTP surface with the following endpoin - `/queries/v1/query-request` -- Submits a SQL query for execution. - `/queries/v1/abort-request` -- Cancels a running query. -The default demo username and password are both `embucket`. JWT tokens have a lifetime of 3 days. Sessions expire after 60 seconds of inactivity. +The default demo username and password both default to `embucket`. JWT tokens have a lifetime of 3 days. Sessions expire after 60 seconds of inactivity. diff --git a/docs/src/content/docs/reference/snowflake.mdx b/docs/src/content/docs/reference/snowflake.mdx index 71a9fecc1..283783e9c 100644 --- a/docs/src/content/docs/reference/snowflake.mdx +++ b/docs/src/content/docs/reference/snowflake.mdx @@ -7,7 +7,7 @@ sidebar: import { Aside, Steps } from '@astrojs/starlight/components'; -Embucket provides Snowflake compatibility through SQL dialect support and REST API compatibility. This page covers core SQL compatibility and API integration for analytical workloads. Snowflake enterprise features, data governance capabilities, and advanced security configurations are out of scope. +Embucket provides Snowflake compatibility through SQL dialect support and REST API compatibility. This page covers core SQL compatibility and API integration for analytical workloads. Snowflake enterprise features, data governance capabilities, and advanced security configurations fall out of scope. Key features: @@ -26,7 +26,7 @@ Embucket exposes a Snowflake v1 REST API that works with any Snowflake client. T ## Query engine -Embucket uses [Apache DataFusion](https://datafusion.apache.org/) as its query engine -- an Apache Arrow-native analytical query engine. The goal is full SQL dialect compatibility with Snowflake. Embucket supports a growing set of Snowflake SQL features. DataFusion also includes some built-in functions that Snowflake does not provide. +Embucket uses [Apache DataFusion](https://datafusion.apache.org/) as its query engine -- an Apache Arrow-native analytical query engine. The project targets full SQL dialect compatibility with Snowflake. Embucket supports a growing set of Snowflake SQL features. DataFusion also includes some built-in functions that Snowflake doesn't provide. ## Compatibility testing @@ -37,17 +37,16 @@ Embucket verifies compatibility through two test methods: ## Architecture differences -Snowflake is a managed analytics database built on FoundationDB and object storage. See the [Snowflake whitepaper](https://www.cs.cmu.edu/~15721-f24/papers/Snowflake.pdf) for details on its architecture. Embucket is an open-source alternative that uses Apache DataFusion, Apache Iceberg, Apache Arrow, and Parquet. +Snowflake operates as a managed analytics database built on FoundationDB and object storage. See the [Snowflake whitepaper](https://www.cs.cmu.edu/~15721-f24/papers/Snowflake.pdf) for details on its architecture. Embucket offers an open-source alternative that uses Apache DataFusion, Apache Iceberg, Apache Arrow, and Parquet. -The key structural differences are: +The key structural differences include: - **Metadata storage** -- Embucket uses external catalogs such as S3 Tables instead of FoundationDB. - **Data format** -- Embucket stores data in Iceberg and Parquet instead of a proprietary format. - **Query execution** -- Embucket runs single-node DataFusion with Arrow in-memory representation instead of a distributed execution engine. ## Current limitations @@ -55,19 +54,19 @@ The key structural differences are: ### Architecture - **Single-node execution** -- Query processing uses only one node's memory and CPU capacity. -- **No distributed parallelism** -- The engine does not distribute queries across nodes. +- **No distributed parallelism** -- The engine doesn't distribute queries across nodes. - **Single writer** -- Only one Embucket instance can write to a table at a time. ### Data types -- **VARIANT** -- Stored as JSON-serialized `TEXT` because Parquet, Iceberg, and Arrow do not support the VARIANT type natively. +- **VARIANT** -- Stored as JSON-serialized `TEXT` because Parquet, Iceberg, and Arrow don't support the VARIANT type natively. - **Numeric type coercion** -- Type promotion and coercion rules differ from Snowflake. - **Timestamp precision** -- Arrow uses a fixed nanosecond precision for timestamps. -- **No collation or charset support** -- All text uses UTF-8 encoding. Snowflake's collation and character set options are not available. +- **No collation or charset support** -- All text uses UTF-8 encoding. Snowflake's collation and character set options lack support. ### Error handling -Error messages do not match Snowflake's error format. Arrow and DataFusion generate errors in their own format. +Error messages don't match Snowflake's error format. Arrow and DataFusion generate errors in their own format. ### Backslash escaping @@ -198,25 +197,25 @@ AVG(A) | SYSTEM$TYPEOF(AVG(A)) ``` ## Timestamp handling -Arrow represents timestamps as 64-bit integers with nanosecond precision. This precision stays constant and cannot change per value. Snowflake uses variable precision for timestamps. +Arrow represents timestamps as 64-bit integers with nanosecond precision. This precision stays constant and can't change per value. Snowflake uses variable precision for timestamps. Timezone handling also differs. Snowflake stores a timezone offset per timestamp value. Embucket stores a timezone offset per column. Query results may vary depending on the session timezone setting. ## Error message format -Arrow and DataFusion generate error messages that do not match Snowflake's error format. If your app parses Snowflake error codes or messages, expect differences when running against Embucket. +Arrow and DataFusion generate error messages that don't match Snowflake's error format. If your app parses Snowflake error codes or messages, expect differences when running against Embucket. ## Development roadmap -The following areas are under active development: +The following areas remain under active development: -- **VARIANT** -- Native storage support when upstream dependencies (Parquet, Iceberg, Arrow) add VARIANT support. +- **VARIANT** -- Native storage support when upstream dependencies -- Parquet, Iceberg, Arrow -- add VARIANT support. - **Numeric types** -- Dynamic precision Decimal types for aggregation functions. - **Timestamps** -- Per-value timezone handling to match Snowflake behavior. - **Error format** -- Align error messages and codes with Snowflake's error format. diff --git a/docs/src/content/docs/reference/troubleshooting.mdx b/docs/src/content/docs/reference/troubleshooting.mdx index cfc31c3ec..7189c4f3f 100644 --- a/docs/src/content/docs/reference/troubleshooting.mdx +++ b/docs/src/content/docs/reference/troubleshooting.mdx @@ -17,11 +17,13 @@ Embucket listens on port 3000 by default. Start the container with the following docker run --name embucket --rm -p 3000:3000 embucket/embucket ``` -Verify that the HTTP surface is reachable at `http://127.0.0.1:3000/`. If port 3000 is already in use, stop the conflicting process or map a different host port (for example, `-p 3001:3000`). +Verify that the HTTP surface responds at `http://127.0.0.1:3000/`. If another process already occupies port 3000, stop it or map a different host port, for example `-p 3001:3000`. ## Snowflake CLI fails with protocol or SSL errors -Embucket does not use TLS for local connections. Set `protocol = "http"` in your Snowflake CLI connection configuration: +{/* vale Google.Parens = NO */} +Embucket doesn't use TLS (Transport Layer Security) for local connections. Set `protocol = "http"` in your Snowflake CLI connection configuration: +{/* vale Google.Parens = YES */} ```toml [connections.local] @@ -41,12 +43,15 @@ The Snowflake CLI defaults to HTTPS, which causes protocol and SSL errors when c ## Authentication fails -Embucket ships with default demo credentials. The username and password are both `embucket`. If you override these values with the `AUTH_DEMO_USER` and `AUTH_DEMO_PASSWORD` environment variables, update your client configuration to match. +Embucket ships with default demo credentials. The username and password both default to `embucket`. If you override these values with the `AUTH_DEMO_USER` and `AUTH_DEMO_PASSWORD` environment variables, update your client configuration to match. + +{/* vale Google.Parens = NO */} +{/* vale Google.Parens = YES */} ## Data doesn't appear @@ -58,27 +63,32 @@ A minimal configuration with no external volumes looks like this: volumes: [] ``` -If data still does not appear, confirm that the credentials and resource ARNs inside your metastore YAML are correct. +If data still doesn't appear, confirm that the credentials and resource ARNs inside your metastore YAML remain correct. + +{/* vale Google.Parens = NO */} +{/* vale Google.Parens = YES */} ## Lambda deploy succeeds but runtime fails When a Lambda deployment succeeds but queries fail at runtime, check the following items in order: 1. **Metastore config** -- Verify that `METASTORE_CONFIG` points to the packaged `metastore.yaml` file inside the Lambda deployment artifact. -2. **Memory and timeout** -- Confirm that the Lambda function has enough memory (default is 3008 MB) and an appropriate timeout (default is 30 seconds). +2. **Memory and timeout** -- Confirm that the Lambda function has enough memory, which defaults to 3008 MB, and an appropriate timeout, which defaults to 30 seconds. 3. **CloudWatch logs** -- Open the log group for your Lambda function and look for error messages that identify the root cause. 4. **Ingress path** -- Confirm that the ingress path matches the client path. Use a Function URL for the Snowflake CLI and direct Lambda invoke for dbt. {/* vale Vale.Terms = NO */} +{/* vale Google.Parens = NO */} +{/* vale Google.Parens = YES */} {/* vale Vale.Terms = YES */} ## dbt can't connect @@ -109,18 +119,21 @@ If the query returns no results, confirm that the metastore configuration on the ## Snowplow package target.type check fails -Some dbt packages, including Snowplow packages, check `target.type == 'snowflake'` and do not recognize the `embucket` target type. Run the compatibility patch script from the Snowplow demo repository to work around this check: +Some dbt packages, including Snowplow packages, check `target.type == 'snowflake'` and don't recognize the `embucket` target type. Run the compatibility patch script from the Snowplow demo repository to work around this check: ```bash ./scripts/patch_snowplow.sh ``` -This is a compatibility workaround. The script modifies package files to accept the `embucket` target type alongside `snowflake`. +This script provides a compatibility workaround. It modifies package files to accept the `embucket` target type alongside `snowflake`. + +{/* vale Google.Parens = NO */} +{/* vale Google.Parens = YES */} ## Sessions feel short-lived diff --git a/docs/src/content/docs/tutorials/snowplow.mdx b/docs/src/content/docs/tutorials/snowplow.mdx index c520cf155..96848a0a7 100644 --- a/docs/src/content/docs/tutorials/snowplow.mdx +++ b/docs/src/content/docs/tutorials/snowplow.mdx @@ -11,7 +11,7 @@ Build a complete web analytics pipeline using Embucket on AWS Lambda with the db ## What you'll build -Snowplow is an open source behavioral data platform that captures granular, event-level web analytics. In this tutorial you connect Snowplow's dbt packages to Embucket and produce three derived tables: +Snowplow provides an open source behavioral data platform that captures granular, event-level web analytics. In this tutorial you connect Snowplow's dbt packages to Embucket and produce three derived tables: - **Page views** -- aggregated metrics for each page view event. - **Sessions** -- session-level summaries stitched from individual events. @@ -27,11 +27,13 @@ Before you begin, make sure you have the following: - Git {/* vale Vale.Terms = NO */} +{/* vale Google.Parens = NO */} +{/* vale Google.Parens = YES */} {/* vale Vale.Terms = YES */} ## Tutorial From 611ffd639116ca3e3d92583cafb01c8bd5ab1ae4 Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Wed, 8 Apr 2026 15:45:07 -0700 Subject: [PATCH 24/25] docs: remove downloaded Vale style packages from git, update gitignore --- .github/styles/.vale-config/2-MDX.ini | 5 - .github/styles/Google/AMPM.yml | 9 - .github/styles/Google/Acronyms.yml | 64 -- .github/styles/Google/Colons.yml | 8 - .github/styles/Google/Contractions.yml | 30 - .github/styles/Google/DateFormat.yml | 9 - .github/styles/Google/Ellipses.yml | 9 - .github/styles/Google/EmDash.yml | 13 - .github/styles/Google/Exclamation.yml | 12 - .github/styles/Google/FirstPerson.yml | 13 - .github/styles/Google/Gender.yml | 9 - .github/styles/Google/GenderBias.yml | 43 -- .github/styles/Google/HeadingPunctuation.yml | 13 - .github/styles/Google/Headings.yml | 29 - .github/styles/Google/Latin.yml | 11 - .github/styles/Google/LyHyphens.yml | 14 - .github/styles/Google/OptionalPlurals.yml | 12 - .github/styles/Google/Ordinal.yml | 7 - .github/styles/Google/OxfordComma.yml | 7 - .github/styles/Google/Parens.yml | 7 - .github/styles/Google/Passive.yml | 184 ----- .github/styles/Google/Periods.yml | 7 - .github/styles/Google/Quotes.yml | 7 - .github/styles/Google/Ranges.yml | 7 - .github/styles/Google/Semicolons.yml | 8 - .github/styles/Google/Slang.yml | 11 - .github/styles/Google/Spacing.yml | 10 - .github/styles/Google/Spelling.yml | 10 - .github/styles/Google/Units.yml | 8 - .github/styles/Google/We.yml | 11 - .github/styles/Google/Will.yml | 7 - .github/styles/Google/WordList.yml | 80 --- .github/styles/Google/meta.json | 4 - .github/styles/Google/vocab.txt | 0 .github/styles/write-good/Cliches.yml | 702 ------------------- .github/styles/write-good/E-Prime.yml | 32 - .github/styles/write-good/Illusions.yml | 11 - .github/styles/write-good/Passive.yml | 183 ----- .github/styles/write-good/README.md | 27 - .github/styles/write-good/So.yml | 5 - .github/styles/write-good/ThereIs.yml | 6 - .github/styles/write-good/TooWordy.yml | 221 ------ .github/styles/write-good/Weasel.yml | 29 - .github/styles/write-good/meta.json | 4 - .gitignore | 9 +- 45 files changed, 4 insertions(+), 1903 deletions(-) delete mode 100644 .github/styles/.vale-config/2-MDX.ini delete mode 100644 .github/styles/Google/AMPM.yml delete mode 100644 .github/styles/Google/Acronyms.yml delete mode 100644 .github/styles/Google/Colons.yml delete mode 100644 .github/styles/Google/Contractions.yml delete mode 100644 .github/styles/Google/DateFormat.yml delete mode 100644 .github/styles/Google/Ellipses.yml delete mode 100644 .github/styles/Google/EmDash.yml delete mode 100644 .github/styles/Google/Exclamation.yml delete mode 100644 .github/styles/Google/FirstPerson.yml delete mode 100644 .github/styles/Google/Gender.yml delete mode 100644 .github/styles/Google/GenderBias.yml delete mode 100644 .github/styles/Google/HeadingPunctuation.yml delete mode 100644 .github/styles/Google/Headings.yml delete mode 100644 .github/styles/Google/Latin.yml delete mode 100644 .github/styles/Google/LyHyphens.yml delete mode 100644 .github/styles/Google/OptionalPlurals.yml delete mode 100644 .github/styles/Google/Ordinal.yml delete mode 100644 .github/styles/Google/OxfordComma.yml delete mode 100644 .github/styles/Google/Parens.yml delete mode 100644 .github/styles/Google/Passive.yml delete mode 100644 .github/styles/Google/Periods.yml delete mode 100644 .github/styles/Google/Quotes.yml delete mode 100644 .github/styles/Google/Ranges.yml delete mode 100644 .github/styles/Google/Semicolons.yml delete mode 100644 .github/styles/Google/Slang.yml delete mode 100644 .github/styles/Google/Spacing.yml delete mode 100644 .github/styles/Google/Spelling.yml delete mode 100644 .github/styles/Google/Units.yml delete mode 100644 .github/styles/Google/We.yml delete mode 100644 .github/styles/Google/Will.yml delete mode 100644 .github/styles/Google/WordList.yml delete mode 100644 .github/styles/Google/meta.json delete mode 100644 .github/styles/Google/vocab.txt delete mode 100644 .github/styles/write-good/Cliches.yml delete mode 100644 .github/styles/write-good/E-Prime.yml delete mode 100644 .github/styles/write-good/Illusions.yml delete mode 100644 .github/styles/write-good/Passive.yml delete mode 100644 .github/styles/write-good/README.md delete mode 100644 .github/styles/write-good/So.yml delete mode 100644 .github/styles/write-good/ThereIs.yml delete mode 100644 .github/styles/write-good/TooWordy.yml delete mode 100644 .github/styles/write-good/Weasel.yml delete mode 100644 .github/styles/write-good/meta.json diff --git a/.github/styles/.vale-config/2-MDX.ini b/.github/styles/.vale-config/2-MDX.ini deleted file mode 100644 index 02ea72b38..000000000 --- a/.github/styles/.vale-config/2-MDX.ini +++ /dev/null @@ -1,5 +0,0 @@ -[*.mdx] -# Exclude: -# -# - Non-JS inline expressions (which cause Acorn to throw) -TokenIgnores = '({#[^\n}]+})(?!`)' diff --git a/.github/styles/Google/AMPM.yml b/.github/styles/Google/AMPM.yml deleted file mode 100644 index 37b49edf8..000000000 --- a/.github/styles/Google/AMPM.yml +++ /dev/null @@ -1,9 +0,0 @@ -extends: existence -message: "Use 'AM' or 'PM' (preceded by a space)." -link: "https://developers.google.com/style/word-list" -level: error -nonword: true -tokens: - - '\d{1,2}[AP]M\b' - - '\d{1,2} ?[ap]m\b' - - '\d{1,2} ?[aApP]\.[mM]\.' diff --git a/.github/styles/Google/Acronyms.yml b/.github/styles/Google/Acronyms.yml deleted file mode 100644 index f41af0189..000000000 --- a/.github/styles/Google/Acronyms.yml +++ /dev/null @@ -1,64 +0,0 @@ -extends: conditional -message: "Spell out '%s', if it's unfamiliar to the audience." -link: 'https://developers.google.com/style/abbreviations' -level: suggestion -ignorecase: false -# Ensures that the existence of 'first' implies the existence of 'second'. -first: '\b([A-Z]{3,5})\b' -second: '(?:\b[A-Z][a-z]+ )+\(([A-Z]{3,5})\)' -# ... with the exception of these: -exceptions: - - API - - ASP - - CLI - - CPU - - CSS - - CSV - - DEBUG - - DOM - - DPI - - FAQ - - GCC - - GDB - - GET - - GPU - - GTK - - GUI - - HTML - - HTTP - - HTTPS - - IDE - - JAR - - JSON - - JSX - - LESS - - LLDB - - NET - - NOTE - - NVDA - - OSS - - PATH - - PDF - - PHP - - POST - - RAM - - REPL - - RSA - - SCM - - SCSS - - SDK - - SQL - - SSH - - SSL - - SVG - - TBD - - TCP - - TODO - - URI - - URL - - USB - - UTF - - XML - - XSS - - YAML - - ZIP diff --git a/.github/styles/Google/Colons.yml b/.github/styles/Google/Colons.yml deleted file mode 100644 index 4a027c307..000000000 --- a/.github/styles/Google/Colons.yml +++ /dev/null @@ -1,8 +0,0 @@ -extends: existence -message: "'%s' should be in lowercase." -link: 'https://developers.google.com/style/colons' -nonword: true -level: warning -scope: sentence -tokens: - - '(?=1.0.0" -} diff --git a/.github/styles/Google/vocab.txt b/.github/styles/Google/vocab.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/styles/write-good/Cliches.yml b/.github/styles/write-good/Cliches.yml deleted file mode 100644 index c95314387..000000000 --- a/.github/styles/write-good/Cliches.yml +++ /dev/null @@ -1,702 +0,0 @@ -extends: existence -message: "Try to avoid using clichés like '%s'." -ignorecase: true -level: warning -tokens: - - a chip off the old block - - a clean slate - - a dark and stormy night - - a far cry - - a fine kettle of fish - - a loose cannon - - a penny saved is a penny earned - - a tough row to hoe - - a word to the wise - - ace in the hole - - acid test - - add insult to injury - - against all odds - - air your dirty laundry - - all fun and games - - all in a day's work - - all talk, no action - - all thumbs - - all your eggs in one basket - - all's fair in love and war - - all's well that ends well - - almighty dollar - - American as apple pie - - an axe to grind - - another day, another dollar - - armed to the teeth - - as luck would have it - - as old as time - - as the crow flies - - at loose ends - - at my wits end - - avoid like the plague - - babe in the woods - - back against the wall - - back in the saddle - - back to square one - - back to the drawing board - - bad to the bone - - badge of honor - - bald faced liar - - ballpark figure - - banging your head against a brick wall - - baptism by fire - - barking up the wrong tree - - bat out of hell - - be all and end all - - beat a dead horse - - beat around the bush - - been there, done that - - beggars can't be choosers - - behind the eight ball - - bend over backwards - - benefit of the doubt - - bent out of shape - - best thing since sliced bread - - bet your bottom dollar - - better half - - better late than never - - better mousetrap - - better safe than sorry - - between a rock and a hard place - - beyond the pale - - bide your time - - big as life - - big cheese - - big fish in a small pond - - big man on campus - - bigger they are the harder they fall - - bird in the hand - - bird's eye view - - birds and the bees - - birds of a feather flock together - - bit the hand that feeds you - - bite the bullet - - bite the dust - - bitten off more than he can chew - - black as coal - - black as pitch - - black as the ace of spades - - blast from the past - - bleeding heart - - blessing in disguise - - blind ambition - - blind as a bat - - blind leading the blind - - blood is thicker than water - - blood sweat and tears - - blow off steam - - blow your own horn - - blushing bride - - boils down to - - bolt from the blue - - bone to pick - - bored stiff - - bored to tears - - bottomless pit - - boys will be boys - - bright and early - - brings home the bacon - - broad across the beam - - broken record - - brought back to reality - - bull by the horns - - bull in a china shop - - burn the midnight oil - - burning question - - burning the candle at both ends - - burst your bubble - - bury the hatchet - - busy as a bee - - by hook or by crook - - call a spade a spade - - called onto the carpet - - calm before the storm - - can of worms - - can't cut the mustard - - can't hold a candle to - - case of mistaken identity - - cat got your tongue - - cat's meow - - caught in the crossfire - - caught red-handed - - checkered past - - chomping at the bit - - cleanliness is next to godliness - - clear as a bell - - clear as mud - - close to the vest - - cock and bull story - - cold shoulder - - come hell or high water - - cool as a cucumber - - cool, calm, and collected - - cost a king's ransom - - count your blessings - - crack of dawn - - crash course - - creature comforts - - cross that bridge when you come to it - - crushing blow - - cry like a baby - - cry me a river - - cry over spilt milk - - crystal clear - - curiosity killed the cat - - cut and dried - - cut through the red tape - - cut to the chase - - cute as a bugs ear - - cute as a button - - cute as a puppy - - cuts to the quick - - dark before the dawn - - day in, day out - - dead as a doornail - - devil is in the details - - dime a dozen - - divide and conquer - - dog and pony show - - dog days - - dog eat dog - - dog tired - - don't burn your bridges - - don't count your chickens - - don't look a gift horse in the mouth - - don't rock the boat - - don't step on anyone's toes - - don't take any wooden nickels - - down and out - - down at the heels - - down in the dumps - - down the hatch - - down to earth - - draw the line - - dressed to kill - - dressed to the nines - - drives me up the wall - - dull as dishwater - - dyed in the wool - - eagle eye - - ear to the ground - - early bird catches the worm - - easier said than done - - easy as pie - - eat your heart out - - eat your words - - eleventh hour - - even the playing field - - every dog has its day - - every fiber of my being - - everything but the kitchen sink - - eye for an eye - - face the music - - facts of life - - fair weather friend - - fall by the wayside - - fan the flames - - feast or famine - - feather your nest - - feathered friends - - few and far between - - fifteen minutes of fame - - filthy vermin - - fine kettle of fish - - fish out of water - - fishing for a compliment - - fit as a fiddle - - fit the bill - - fit to be tied - - flash in the pan - - flat as a pancake - - flip your lid - - flog a dead horse - - fly by night - - fly the coop - - follow your heart - - for all intents and purposes - - for the birds - - for what it's worth - - force of nature - - force to be reckoned with - - forgive and forget - - fox in the henhouse - - free and easy - - free as a bird - - fresh as a daisy - - full steam ahead - - fun in the sun - - garbage in, garbage out - - gentle as a lamb - - get a kick out of - - get a leg up - - get down and dirty - - get the lead out - - get to the bottom of - - get your feet wet - - gets my goat - - gilding the lily - - give and take - - go against the grain - - go at it tooth and nail - - go for broke - - go him one better - - go the extra mile - - go with the flow - - goes without saying - - good as gold - - good deed for the day - - good things come to those who wait - - good time was had by all - - good times were had by all - - greased lightning - - greek to me - - green thumb - - green-eyed monster - - grist for the mill - - growing like a weed - - hair of the dog - - hand to mouth - - happy as a clam - - happy as a lark - - hasn't a clue - - have a nice day - - have high hopes - - have the last laugh - - haven't got a row to hoe - - head honcho - - head over heels - - hear a pin drop - - heard it through the grapevine - - heart's content - - heavy as lead - - hem and haw - - high and dry - - high and mighty - - high as a kite - - hit paydirt - - hold your head up high - - hold your horses - - hold your own - - hold your tongue - - honest as the day is long - - horns of a dilemma - - horse of a different color - - hot under the collar - - hour of need - - I beg to differ - - icing on the cake - - if the shoe fits - - if the shoe were on the other foot - - in a jam - - in a jiffy - - in a nutshell - - in a pig's eye - - in a pinch - - in a word - - in hot water - - in the gutter - - in the nick of time - - in the thick of it - - in your dreams - - it ain't over till the fat lady sings - - it goes without saying - - it takes all kinds - - it takes one to know one - - it's a small world - - it's only a matter of time - - ivory tower - - Jack of all trades - - jockey for position - - jog your memory - - joined at the hip - - judge a book by its cover - - jump down your throat - - jump in with both feet - - jump on the bandwagon - - jump the gun - - jump to conclusions - - just a hop, skip, and a jump - - just the ticket - - justice is blind - - keep a stiff upper lip - - keep an eye on - - keep it simple, stupid - - keep the home fires burning - - keep up with the Joneses - - keep your chin up - - keep your fingers crossed - - kick the bucket - - kick up your heels - - kick your feet up - - kid in a candy store - - kill two birds with one stone - - kiss of death - - knock it out of the park - - knock on wood - - knock your socks off - - know him from Adam - - know the ropes - - know the score - - knuckle down - - knuckle sandwich - - knuckle under - - labor of love - - ladder of success - - land on your feet - - lap of luxury - - last but not least - - last hurrah - - last-ditch effort - - law of the jungle - - law of the land - - lay down the law - - leaps and bounds - - let sleeping dogs lie - - let the cat out of the bag - - let the good times roll - - let your hair down - - let's talk turkey - - letter perfect - - lick your wounds - - lies like a rug - - life's a bitch - - life's a grind - - light at the end of the tunnel - - lighter than a feather - - lighter than air - - like clockwork - - like father like son - - like taking candy from a baby - - like there's no tomorrow - - lion's share - - live and learn - - live and let live - - long and short of it - - long lost love - - look before you leap - - look down your nose - - look what the cat dragged in - - looking a gift horse in the mouth - - looks like death warmed over - - loose cannon - - lose your head - - lose your temper - - loud as a horn - - lounge lizard - - loved and lost - - low man on the totem pole - - luck of the draw - - luck of the Irish - - make hay while the sun shines - - make money hand over fist - - make my day - - make the best of a bad situation - - make the best of it - - make your blood boil - - man of few words - - man's best friend - - mark my words - - meaningful dialogue - - missed the boat on that one - - moment in the sun - - moment of glory - - moment of truth - - money to burn - - more power to you - - more than one way to skin a cat - - movers and shakers - - moving experience - - naked as a jaybird - - naked truth - - neat as a pin - - needle in a haystack - - needless to say - - neither here nor there - - never look back - - never say never - - nip and tuck - - nip it in the bud - - no guts, no glory - - no love lost - - no pain, no gain - - no skin off my back - - no stone unturned - - no time like the present - - no use crying over spilled milk - - nose to the grindstone - - not a hope in hell - - not a minute's peace - - not in my backyard - - not playing with a full deck - - not the end of the world - - not written in stone - - nothing to sneeze at - - nothing ventured nothing gained - - now we're cooking - - off the top of my head - - off the wagon - - off the wall - - old hat - - older and wiser - - older than dirt - - older than Methuselah - - on a roll - - on cloud nine - - on pins and needles - - on the bandwagon - - on the money - - on the nose - - on the rocks - - on the spot - - on the tip of my tongue - - on the wagon - - on thin ice - - once bitten, twice shy - - one bad apple doesn't spoil the bushel - - one born every minute - - one brick short - - one foot in the grave - - one in a million - - one red cent - - only game in town - - open a can of worms - - open and shut case - - open the flood gates - - opportunity doesn't knock twice - - out of pocket - - out of sight, out of mind - - out of the frying pan into the fire - - out of the woods - - out on a limb - - over a barrel - - over the hump - - pain and suffering - - pain in the - - panic button - - par for the course - - part and parcel - - party pooper - - pass the buck - - patience is a virtue - - pay through the nose - - penny pincher - - perfect storm - - pig in a poke - - pile it on - - pillar of the community - - pin your hopes on - - pitter patter of little feet - - plain as day - - plain as the nose on your face - - play by the rules - - play your cards right - - playing the field - - playing with fire - - pleased as punch - - plenty of fish in the sea - - point with pride - - poor as a church mouse - - pot calling the kettle black - - pretty as a picture - - pull a fast one - - pull your punches - - pulling your leg - - pure as the driven snow - - put it in a nutshell - - put one over on you - - put the cart before the horse - - put the pedal to the metal - - put your best foot forward - - put your foot down - - quick as a bunny - - quick as a lick - - quick as a wink - - quick as lightning - - quiet as a dormouse - - rags to riches - - raining buckets - - raining cats and dogs - - rank and file - - rat race - - reap what you sow - - red as a beet - - red herring - - reinvent the wheel - - rich and famous - - rings a bell - - ripe old age - - ripped me off - - rise and shine - - road to hell is paved with good intentions - - rob Peter to pay Paul - - roll over in the grave - - rub the wrong way - - ruled the roost - - running in circles - - sad but true - - sadder but wiser - - salt of the earth - - scared stiff - - scared to death - - sealed with a kiss - - second to none - - see eye to eye - - seen the light - - seize the day - - set the record straight - - set the world on fire - - set your teeth on edge - - sharp as a tack - - shoot for the moon - - shoot the breeze - - shot in the dark - - shoulder to the wheel - - sick as a dog - - sigh of relief - - signed, sealed, and delivered - - sink or swim - - six of one, half a dozen of another - - skating on thin ice - - slept like a log - - slinging mud - - slippery as an eel - - slow as molasses - - smart as a whip - - smooth as a baby's bottom - - sneaking suspicion - - snug as a bug in a rug - - sow wild oats - - spare the rod, spoil the child - - speak of the devil - - spilled the beans - - spinning your wheels - - spitting image of - - spoke with relish - - spread like wildfire - - spring to life - - squeaky wheel gets the grease - - stands out like a sore thumb - - start from scratch - - stick in the mud - - still waters run deep - - stitch in time - - stop and smell the roses - - straight as an arrow - - straw that broke the camel's back - - strong as an ox - - stubborn as a mule - - stuff that dreams are made of - - stuffed shirt - - sweating blood - - sweating bullets - - take a load off - - take one for the team - - take the bait - - take the bull by the horns - - take the plunge - - takes one to know one - - takes two to tango - - the more the merrier - - the real deal - - the real McCoy - - the red carpet treatment - - the same old story - - there is no accounting for taste - - thick as a brick - - thick as thieves - - thin as a rail - - think outside of the box - - third time's the charm - - this day and age - - this hurts me worse than it hurts you - - this point in time - - three sheets to the wind - - through thick and thin - - throw in the towel - - tie one on - - tighter than a drum - - time and time again - - time is of the essence - - tip of the iceberg - - tired but happy - - to coin a phrase - - to each his own - - to make a long story short - - to the best of my knowledge - - toe the line - - tongue in cheek - - too good to be true - - too hot to handle - - too numerous to mention - - touch with a ten foot pole - - tough as nails - - trial and error - - trials and tribulations - - tried and true - - trip down memory lane - - twist of fate - - two cents worth - - two peas in a pod - - ugly as sin - - under the counter - - under the gun - - under the same roof - - under the weather - - until the cows come home - - unvarnished truth - - up the creek - - uphill battle - - upper crust - - upset the applecart - - vain attempt - - vain effort - - vanquish the enemy - - vested interest - - waiting for the other shoe to drop - - wakeup call - - warm welcome - - watch your p's and q's - - watch your tongue - - watching the clock - - water under the bridge - - weather the storm - - weed them out - - week of Sundays - - went belly up - - wet behind the ears - - what goes around comes around - - what you see is what you get - - when it rains, it pours - - when push comes to shove - - when the cat's away - - when the going gets tough, the tough get going - - white as a sheet - - whole ball of wax - - whole hog - - whole nine yards - - wild goose chase - - will wonders never cease? - - wisdom of the ages - - wise as an owl - - wolf at the door - - words fail me - - work like a dog - - world weary - - worst nightmare - - worth its weight in gold - - wrong side of the bed - - yanking your chain - - yappy as a dog - - years young - - you are what you eat - - you can run but you can't hide - - you only live once - - you're the boss - - young and foolish - - young and vibrant diff --git a/.github/styles/write-good/E-Prime.yml b/.github/styles/write-good/E-Prime.yml deleted file mode 100644 index 074a102b2..000000000 --- a/.github/styles/write-good/E-Prime.yml +++ /dev/null @@ -1,32 +0,0 @@ -extends: existence -message: "Try to avoid using '%s'." -ignorecase: true -level: suggestion -tokens: - - am - - are - - aren't - - be - - been - - being - - he's - - here's - - here's - - how's - - i'm - - is - - isn't - - it's - - she's - - that's - - there's - - they're - - was - - wasn't - - we're - - were - - weren't - - what's - - where's - - who's - - you're diff --git a/.github/styles/write-good/Illusions.yml b/.github/styles/write-good/Illusions.yml deleted file mode 100644 index b4f132185..000000000 --- a/.github/styles/write-good/Illusions.yml +++ /dev/null @@ -1,11 +0,0 @@ -extends: repetition -message: "'%s' is repeated!" -level: warning -alpha: true -action: - name: edit - params: - - truncate - - " " -tokens: - - '[^\s]+' diff --git a/.github/styles/write-good/Passive.yml b/.github/styles/write-good/Passive.yml deleted file mode 100644 index f472cb904..000000000 --- a/.github/styles/write-good/Passive.yml +++ /dev/null @@ -1,183 +0,0 @@ -extends: existence -message: "'%s' may be passive voice. Use active voice if you can." -ignorecase: true -level: warning -raw: - - \b(am|are|were|being|is|been|was|be)\b\s* -tokens: - - '[\w]+ed' - - awoken - - beat - - become - - been - - begun - - bent - - beset - - bet - - bid - - bidden - - bitten - - bled - - blown - - born - - bought - - bound - - bred - - broadcast - - broken - - brought - - built - - burnt - - burst - - cast - - caught - - chosen - - clung - - come - - cost - - crept - - cut - - dealt - - dived - - done - - drawn - - dreamt - - driven - - drunk - - dug - - eaten - - fallen - - fed - - felt - - fit - - fled - - flown - - flung - - forbidden - - foregone - - forgiven - - forgotten - - forsaken - - fought - - found - - frozen - - given - - gone - - gotten - - ground - - grown - - heard - - held - - hidden - - hit - - hung - - hurt - - kept - - knelt - - knit - - known - - laid - - lain - - leapt - - learnt - - led - - left - - lent - - let - - lighted - - lost - - made - - meant - - met - - misspelt - - mistaken - - mown - - overcome - - overdone - - overtaken - - overthrown - - paid - - pled - - proven - - put - - quit - - read - - rid - - ridden - - risen - - run - - rung - - said - - sat - - sawn - - seen - - sent - - set - - sewn - - shaken - - shaven - - shed - - shod - - shone - - shorn - - shot - - shown - - shrunk - - shut - - slain - - slept - - slid - - slit - - slung - - smitten - - sold - - sought - - sown - - sped - - spent - - spilt - - spit - - split - - spoken - - spread - - sprung - - spun - - stolen - - stood - - stridden - - striven - - struck - - strung - - stuck - - stung - - stunk - - sung - - sunk - - swept - - swollen - - sworn - - swum - - swung - - taken - - taught - - thought - - thrived - - thrown - - thrust - - told - - torn - - trodden - - understood - - upheld - - upset - - wed - - wept - - withheld - - withstood - - woken - - won - - worn - - wound - - woven - - written - - wrung diff --git a/.github/styles/write-good/README.md b/.github/styles/write-good/README.md deleted file mode 100644 index 3edcc9b37..000000000 --- a/.github/styles/write-good/README.md +++ /dev/null @@ -1,27 +0,0 @@ -Based on [write-good](https://github.com/btford/write-good). - -> Naive linter for English prose for developers who can't write good and wanna learn to do other stuff good too. - -``` -The MIT License (MIT) - -Copyright (c) 2014 Brian Ford - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` diff --git a/.github/styles/write-good/So.yml b/.github/styles/write-good/So.yml deleted file mode 100644 index e57f099dc..000000000 --- a/.github/styles/write-good/So.yml +++ /dev/null @@ -1,5 +0,0 @@ -extends: existence -message: "Don't start a sentence with '%s'." -level: error -raw: - - '(?:[;-]\s)so[\s,]|\bSo[\s,]' diff --git a/.github/styles/write-good/ThereIs.yml b/.github/styles/write-good/ThereIs.yml deleted file mode 100644 index 8b82e8f6c..000000000 --- a/.github/styles/write-good/ThereIs.yml +++ /dev/null @@ -1,6 +0,0 @@ -extends: existence -message: "Don't start a sentence with '%s'." -ignorecase: false -level: error -raw: - - '(?:[;-]\s)There\s(is|are)|\bThere\s(is|are)\b' diff --git a/.github/styles/write-good/TooWordy.yml b/.github/styles/write-good/TooWordy.yml deleted file mode 100644 index 275701b19..000000000 --- a/.github/styles/write-good/TooWordy.yml +++ /dev/null @@ -1,221 +0,0 @@ -extends: existence -message: "'%s' is too wordy." -ignorecase: true -level: warning -tokens: - - a number of - - abundance - - accede to - - accelerate - - accentuate - - accompany - - accomplish - - accorded - - accrue - - acquiesce - - acquire - - additional - - adjacent to - - adjustment - - admissible - - advantageous - - adversely impact - - advise - - aforementioned - - aggregate - - aircraft - - all of - - all things considered - - alleviate - - allocate - - along the lines of - - already existing - - alternatively - - amazing - - ameliorate - - anticipate - - apparent - - appreciable - - as a matter of fact - - as a means of - - as far as I'm concerned - - as of yet - - as to - - as yet - - ascertain - - assistance - - at the present time - - at this time - - attain - - attributable to - - authorize - - because of the fact that - - belated - - benefit from - - bestow - - by means of - - by virtue of - - by virtue of the fact that - - cease - - close proximity - - commence - - comply with - - concerning - - consequently - - consolidate - - constitutes - - demonstrate - - depart - - designate - - discontinue - - due to the fact that - - each and every - - economical - - eliminate - - elucidate - - employ - - endeavor - - enumerate - - equitable - - equivalent - - evaluate - - evidenced - - exclusively - - expedite - - expend - - expiration - - facilitate - - factual evidence - - feasible - - finalize - - first and foremost - - for all intents and purposes - - for the most part - - for the purpose of - - forfeit - - formulate - - have a tendency to - - honest truth - - however - - if and when - - impacted - - implement - - in a manner of speaking - - in a timely manner - - in a very real sense - - in accordance with - - in addition - - in all likelihood - - in an effort to - - in between - - in excess of - - in lieu of - - in light of the fact that - - in many cases - - in my opinion - - in order to - - in regard to - - in some instances - - in terms of - - in the case of - - in the event that - - in the final analysis - - in the nature of - - in the near future - - in the process of - - inception - - incumbent upon - - indicate - - indication - - initiate - - irregardless - - is applicable to - - is authorized to - - is responsible for - - it is - - it is essential - - it seems that - - it was - - magnitude - - maximum - - methodology - - minimize - - minimum - - modify - - monitor - - multiple - - necessitate - - nevertheless - - not certain - - not many - - not often - - not unless - - not unlike - - notwithstanding - - null and void - - numerous - - objective - - obligate - - obtain - - on the contrary - - on the other hand - - one particular - - optimum - - overall - - owing to the fact that - - participate - - particulars - - pass away - - pertaining to - - point in time - - portion - - possess - - preclude - - previously - - prior to - - prioritize - - procure - - proficiency - - provided that - - purchase - - put simply - - readily apparent - - refer back - - regarding - - relocate - - remainder - - remuneration - - requirement - - reside - - residence - - retain - - satisfy - - shall - - should you wish - - similar to - - solicit - - span across - - strategize - - subsequent - - substantial - - successfully complete - - sufficient - - terminate - - the month of - - the point I am trying to make - - therefore - - time period - - took advantage of - - transmit - - transpire - - type of - - until such time as - - utilization - - utilize - - validate - - various different - - what I mean to say is - - whether or not - - with respect to - - with the exception of - - witnessed diff --git a/.github/styles/write-good/Weasel.yml b/.github/styles/write-good/Weasel.yml deleted file mode 100644 index d1d90a7bc..000000000 --- a/.github/styles/write-good/Weasel.yml +++ /dev/null @@ -1,29 +0,0 @@ -extends: existence -message: "'%s' is a weasel word!" -ignorecase: true -level: warning -tokens: - - clearly - - completely - - exceedingly - - excellent - - extremely - - fairly - - huge - - interestingly - - is a number - - largely - - mostly - - obviously - - quite - - relatively - - remarkably - - several - - significantly - - substantially - - surprisingly - - tiny - - usually - - various - - vast - - very diff --git a/.github/styles/write-good/meta.json b/.github/styles/write-good/meta.json deleted file mode 100644 index a115d2886..000000000 --- a/.github/styles/write-good/meta.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "feed": "https://github.com/errata-ai/write-good/releases.atom", - "vale_version": ">=1.0.0" -} diff --git a/.gitignore b/.gitignore index 32751a99c..0a790c351 100644 --- a/.gitignore +++ b/.gitignore @@ -7,11 +7,10 @@ data/ .env metastore.yaml -# Ignore only the root styles directory; allow .github/styles for Vale -/styles/ -!/styles/config -!.github/styles/ -!.github/styles/** +# Vale styles — only keep custom vocabulary, ignore downloaded packages +.github/styles/Google/ +.github/styles/write-good/ +.github/styles/.vale-config/ **/*.rs.bk .DS_Store From d8ebcd315ce9b873356a007162160c53484c0829 Mon Sep 17 00:00:00 2001 From: Sergei Turukin Date: Wed, 8 Apr 2026 15:45:30 -0700 Subject: [PATCH 25/25] docs: remove superpowers specs/plans from git, add to gitignore --- .gitignore | 1 + .../plans/2026-04-08-docs-restructure.md | 949 ------------------ .../2026-04-08-docs-restructure-design.md | 342 ------- 3 files changed, 1 insertion(+), 1291 deletions(-) delete mode 100644 docs/superpowers/plans/2026-04-08-docs-restructure.md delete mode 100644 docs/superpowers/specs/2026-04-08-docs-restructure-design.md diff --git a/.gitignore b/.gitignore index 0a790c351..a8774ed3e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ metastore.yaml .log metastore.yaml +docs/superpowers/ diff --git a/docs/superpowers/plans/2026-04-08-docs-restructure.md b/docs/superpowers/plans/2026-04-08-docs-restructure.md deleted file mode 100644 index 45507ae18..000000000 --- a/docs/superpowers/plans/2026-04-08-docs-restructure.md +++ /dev/null @@ -1,949 +0,0 @@ -# Docs Restructure Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Restructure Embucket public docs from a confusing repo-centric layout into a clean product-docs structure: Getting Started, Deploy, Connect, Tutorials, Reference. - -**Architecture:** Delete 7 old pages and 2 old directories; create 5 new directories with 9 pages. Update astro sidebar config, CI workflow, and package.json. Each page is written by a docs-writer agent, reviewed by a docs-reviewer agent, then stranger-audited at the end. README.md gets a full rewrite. - -**Tech Stack:** Astro Starlight, MDX, Vale linter (Google + write-good styles), pnpm - -**Spec:** `docs/superpowers/specs/2026-04-08-docs-restructure-design.md` - ---- - -### Task 1: Scaffold directories and update config - -**Files:** - -- Modify: `docs/astro.config.mjs` -- Modify: `docs/package.json` -- Modify: `.github/workflows/docs-ci.yml` -- Delete: `docs/scripts/validate-docs-smoke.mjs` -- Delete: `docs/src/content/docs/essentials/` (entire directory) -- Delete: `docs/src/content/docs/guides/` (entire directory) -- Delete: `docs/src/content/docs/development/` (entire directory) -- Create: `docs/src/content/docs/getting-started/` -- Create: `docs/src/content/docs/deploy/` -- Create: `docs/src/content/docs/connect/` -- Create: `docs/src/content/docs/tutorials/` -- Create: `docs/src/content/docs/reference/` - -- [ ] **Step 1: Create new directories** - -```bash -mkdir -p docs/src/content/docs/getting-started -mkdir -p docs/src/content/docs/deploy -mkdir -p docs/src/content/docs/connect -mkdir -p docs/src/content/docs/tutorials -mkdir -p docs/src/content/docs/reference -``` - -- [ ] **Step 2: Copy images to new locations** - -Images referenced by pages need to move with them: - -```bash -# Architecture diagram stays with architecture page -cp docs/src/content/docs/essentials/architecture.png docs/src/content/docs/reference/architecture.png -# Quick start UI screenshot -cp docs/src/content/docs/essentials/quick-start-ui.png docs/src/content/docs/getting-started/quick-start-ui.png -# S3 tables query screenshot goes to deploy (absorbed into Lambda page) -cp docs/src/content/docs/guides/s3-tables-query.png docs/src/content/docs/deploy/s3-tables-query.png -# Copy any create-volume images -cp docs/src/content/docs/essentials/create-volume-*.png docs/src/content/docs/deploy/ 2>/dev/null || true -``` - -- [ ] **Step 3: Delete old directories and files** - -```bash -rm -rf docs/src/content/docs/essentials -rm -rf docs/src/content/docs/guides -rm -rf docs/src/content/docs/development -rm docs/scripts/validate-docs-smoke.mjs -``` - -- [ ] **Step 4: Update astro.config.mjs sidebar** - -Replace the sidebar config in `docs/astro.config.mjs`: - -```javascript -sidebar: [ - { - label: 'Getting Started', - autogenerate: { directory: 'getting-started' }, - }, - { - label: 'Deploy', - autogenerate: { directory: 'deploy' }, - }, - { - label: 'Connect', - autogenerate: { directory: 'connect' }, - }, - { - label: 'Tutorials', - autogenerate: { directory: 'tutorials' }, - }, - { - label: 'Reference', - autogenerate: { directory: 'reference' }, - }, -], -``` - -Also update the root redirect: - -```javascript -redirects: { - '/': '/getting-started/quick-start/', -}, -``` - -- [ ] **Step 5: Remove smoke script from package.json** - -Remove the `"smoke"` entry from `scripts` in `docs/package.json`: - -```json -"smoke": "node ./scripts/validate-docs-smoke.mjs" -``` - -Delete that line. - -- [ ] **Step 6: Update CI workflow** - -In `.github/workflows/docs-ci.yml`: - -1. Delete the entire `smoke` job (lines 54-89) -2. In the `build` job, change `needs: [format, smoke]` to `needs: [format]` - -- [ ] **Step 7: Verify scaffold builds** - -```bash -cd docs && pnpm install && pnpm check -``` - -Expected: passes (no content files yet, but config is valid) - -- [ ] **Step 8: Commit scaffold** - -```bash -git add -A docs/ .github/workflows/docs-ci.yml -git commit -m "docs: scaffold new directory structure, remove smoke checks" -``` - ---- - -### Task 2: Write Quick Start page - -**Files:** - -- Create: `docs/src/content/docs/getting-started/quick-start.mdx` - -**Source material:** - -- Old file: `essentials/quick-start.mdx` (read from git: `git show HEAD~1:docs/src/content/docs/essentials/quick-start.mdx`) -- Spec section: "Getting Started / Quick Start" - -- [ ] **Step 1: Draft page with docs-writer agent** - -Dispatch a docs-writer agent with this prompt: - -> Write a Starlight MDX page for `docs/src/content/docs/getting-started/quick-start.mdx`. -> -> **Frontmatter:** title "Quick Start", description "Try Embucket locally with Docker and run your first query.", sidebar order 0. -> -> **Purpose:** Try Embucket locally in 2 minutes before deploying to AWS. -> -> **Content to include (in order):** -> -> 1. One-line intro: "Try Embucket locally before deploying to AWS." -> 2. Step 1: Start Embucket — `docker run --name embucket --rm -p 3000:3000 embucket/embucket` with expected startup log showing `Listening on http://0.0.0.0:3000` -> 3. Step 2: Configure Snowflake CLI — install with `python -m pip install snowflake-cli`, find config with `snow --info`, add TOML connection block (host=localhost, region=us-east-2, port=3000, protocol=http, database=embucket, schema=public, warehouse=em.wh, account=acc.local, user=embucket, password=embucket). Test with `snow connection test -c local`. -> 4. Step 3: Run first query — `snow sql -c local -q "select dateadd(day, -1, current_timestamp()) as yesterday;"` with expected output. -> 5. Next steps section with exactly 3 links: Deploy to AWS Lambda (`/deploy/aws-lambda/`), Connect with Snowflake CLI (`/connect/snowflake-cli/`), Connect with dbt (`/connect/dbt/`) -> -> **Rules:** -> -> - No "Owner/Last reviewed" blockquotes -> - No web UI mention -> - No hedging language ("test and evaluation", "not production-ready") -> - No self-hosted binary references -> - Use Starlight `Steps` component for the numbered steps -> - Import `{ Aside, Steps }` from `@astrojs/starlight/components` -> - Write as clean product documentation -> - Vale linter uses Google style guide — use second person ("you"), active voice, present tense - -- [ ] **Step 2: Review page with docs-reviewer agent** - -Dispatch a docs-reviewer agent with this prompt: - -> Review `docs/src/content/docs/getting-started/quick-start.mdx` against these criteria: -> -> 1. No "Owner:", "Last reviewed:", or internal metadata visible -> 2. No repo-centric language ("the repo", "current docs", "currently documented") -> 3. No hedging ("test and evaluation path", "not production-ready", "the confusion around") -> 4. No web UI or HTTP surface mention -> 5. No self-hosted binary references -> 6. Next steps has exactly 3 links to /deploy/aws-lambda/, /connect/snowflake-cli/, /connect/dbt/ -> 7. Uses Starlight Steps component -> 8. All code blocks have language tags -> 9. Would pass Vale with Google style (second person, active voice, present tense) -> Report issues as a numbered list. If clean, say "PASS". - -- [ ] **Step 3: Fix any review findings** - -- [ ] **Step 4: Verify build** - -```bash -cd docs && pnpm build 2>&1 | head -50 -``` - -Expected: build succeeds, no broken links for this page - -- [ ] **Step 5: Commit** - -```bash -git add docs/src/content/docs/getting-started/quick-start.mdx -git commit -m "docs: add Quick Start page" -``` - ---- - -### Task 3: Write AWS Lambda deployment page - -**Files:** - -- Create: `docs/src/content/docs/deploy/aws-lambda.mdx` - -**Source material:** - -- Old files (from git): `guides/aws-lambda.mdx`, `guides/s3-tables.mdx`, `essentials/runtime-modes.mdx` -- `crates/embucket-lambda/Cargo.toml` — Lambda deploy metadata (memory 3008, timeout 30) -- `crates/embucket-lambda/Makefile` — deploy targets and variables -- `crates/state-store/README.md` — DynamoDB setup, env vars, table schema - -- [ ] **Step 1: Draft page with docs-writer agent** - -Dispatch a docs-writer agent with this prompt: - -> Write a Starlight MDX page for `docs/src/content/docs/deploy/aws-lambda.mdx`. -> -> **Frontmatter:** title "AWS Lambda", description "Deploy Embucket to AWS Lambda with S3 Tables storage.", sidebar order 0. -> -> **Purpose:** Single page covering the full path from zero to a running Embucket Lambda with S3 Tables. -> -> **Content sections (in order):** -> -> 1. **Intro** (2 sentences): Embucket runs as an AWS Lambda function using S3 Tables (Apache Iceberg) for storage. This guide covers the full deployment from creating your S3 table bucket through verifying a working query. -> 2. **Prerequisites**: Rust toolchain, `cargo-lambda` (`cargo install cargo-lambda`), AWS CLI installed and configured, AWS credentials with Lambda/S3Tables/IAM permissions. -> 3. **Create an S3 table bucket**: Use `aws s3tables create-table-bucket --name my-table-bucket --region us-east-2`. Show the JSON response with ARN. Tell user to save bucket name, region, and ARN. -> 4. **Configure the metastore**: Create a YAML config file at `config/metastore.yaml`: -> -> ```yaml -> volumes: -> - ident: embucket -> type: s3-tables -> database: demo -> credentials: -> credential_type: access_key -> aws-access-key-id: ACCESS_KEY -> aws-secret-access-key: SECRET_ACCESS_KEY -> arn: arn:aws:s3tables:us-east-2:123456789012:bucket/my-table-bucket -> ``` -> -> Add an Aside tip to replace placeholders with real values. -> -> 5. **Build and deploy**: From repo root: `make -C crates/embucket-lambda deploy`. Show env file variant: `ENV_FILE=config/.env.lambda make -C crates/embucket-lambda deploy`. Show role ARN variant: `AWS_LAMBDA_ROLE_ARN=arn:aws:iam::123456789012:role/embucket-lambda-role make -C crates/embucket-lambda deploy`. Mention the Makefile variables: `FUNCTION_NAME` (default: embucket-lambda), `ENV_FILE`, `AWS_LAMBDA_ROLE_ARN`, `FEATURES`, `LAYERS`, `WITH_OTEL_CONFIG`. -> 6. **Verify the deployment**: `make -C crates/embucket-lambda verify` (runs `snow sql -c lambda -q "SELECT 1 as test_column"`). Also `make -C crates/embucket-lambda logs` to tail CloudWatch. Show the direct curl login-request example for HTTP-level validation. -> 7. **IAM and access**: Two concerns: (a) deployer permissions — identity running `cargo lambda deploy`; (b) execution role — the Lambda function itself. Mention that `dbt-embucket` uses AWS credentials to invoke Lambda directly by ARN. Plan for: permission to deploy/update, permission to read logs, permission for dbt clients to invoke. -> 8. **Lambda sizing**: Default memory is 3008 MB, timeout is 30 seconds (set in Cargo.toml deploy metadata). 3008 MB is the standard Lambda maximum — increasing beyond that requires an AWS support ticket (up to 10 GB). Override memory/timeout through `cargo lambda deploy` flags or AWS Console. For larger datasets or complex queries, consider requesting a memory increase. Tracing is set to Active by default. -> 9. **Statestore (optional)**: For persistent query state across invocations, build with the state-store feature flag: `FEATURES=state-store-query make -C crates/embucket-lambda deploy`. Requires a DynamoDB table. Show the `aws dynamodb create-table` command creating table `embucket-statestore` with PK/SK keys and GSIs for query_id, request_id, session_id. Environment variables: `STATESTORE_TABLE_NAME` (default: embucket-statestore), `STATESTORE_DYNAMODB_ENDPOINT` (for local testing: http://localhost:8000), `AWS_DDB_ACCESS_KEY_ID`, `AWS_DDB_SECRET_ACCESS_KEY`, `AWS_DDB_SESSION_TOKEN` (optional, for temporary credentials). -> 10. **Production ingress**: For production traffic, keep the Lambda private and put an API Gateway in front. Show the CloudFormation skeleton: -> -> ```yaml -> Parameters: -> LambdaFunctionName: -> Type: String -> Default: embucket-lambda -> VpcId: -> Type: AWS::EC2::VPC::Id -> SubnetIds: -> Type: List -> VpcCidr: -> Type: String -> Default: 10.0.0.0/16 -> -> Resources: -> ExecuteApiVpcEndpoint: -> Type: AWS::EC2::VPCEndpoint -> PrivateApi: -> Type: AWS::ApiGateway::RestApi -> LambdaInvokePermission: -> Type: AWS::Lambda::Permission -> ``` -> -> Note: provisions a private API Gateway, VPC endpoint for execute-api, Lambda proxy integration, stage named v1. Caution Aside: a public Function URL is acceptable for testing but should not be used for production traffic. -> -> 11. **Rollback and redeploy**: Keep previous env file and metastore config in version control. Redeploy with previous config if a change causes regression. Re-run verification after rollback. -> 12. **Cleanup**: `aws lambda delete-function-url-config --function-name embucket-lambda` to remove Function URL. Also clean up Lambda function, CloudWatch log group, API Gateway and VPC endpoint if used, telemetry layers. -> 13. **Troubleshooting** (inline at bottom): Deploy succeeds but queries fail → check METASTORE_CONFIG points to real file. dbt cannot connect → check AWS credentials and EMBUCKET_FUNCTION_ARN. Timeouts or truncated responses → review timeout and memory settings. No useful traces → verify RUST_LOG, TRACING_LEVEL, OTEL config. -> -> **Rules:** -> -> - No "Owner/Last reviewed" blockquotes -> - No repo-centric language -> - Import `{ Aside, Steps }` from `@astrojs/starlight/components` -> - Use Steps component for the main deployment flow -> - Vale Google style: second person, active voice, present tense -> - This is the most important page in the docs — it must be thorough and clear - -- [ ] **Step 2: Review page with docs-reviewer agent** - -Dispatch a docs-reviewer agent: - -> Review `docs/src/content/docs/deploy/aws-lambda.mdx` against the spec at `docs/superpowers/specs/2026-04-08-docs-restructure-design.md`, section "Deploy / AWS Lambda". Check: -> -> 1. All 12 content sections present (prerequisites through troubleshooting) -> 2. S3 table bucket creation absorbed from old s3-tables guide -> 3. Statestore section with feature flag `-F state-store` and DynamoDB setup -> 4. Lambda sizing section mentioning 3008 MB default and AWS support ticket for >3008 -> 5. No "Owner/Last reviewed", no repo-centric language, no internal commentary -> 6. All code blocks have language tags -> 7. Asides used appropriately (caution for Function URL, tips for config) -> Report issues as a numbered list. If clean, say "PASS". - -- [ ] **Step 3: Fix any review findings** - -- [ ] **Step 4: Verify build** - -```bash -cd docs && pnpm build 2>&1 | head -50 -``` - -- [ ] **Step 5: Commit** - -```bash -git add docs/src/content/docs/deploy/aws-lambda.mdx -git commit -m "docs: add AWS Lambda deployment page" -``` - ---- - -### Task 4: Write Configuration page - -**Files:** - -- Create: `docs/src/content/docs/deploy/configuration.mdx` - -**Source material:** - -- Old file (from git): `essentials/configuration.mdx` -- `crates/embucketd/src/cli.rs` — for flag/env var reference -- `crates/embucket-lambda/Makefile` — Makefile variables -- `crates/state-store/README.md` — statestore env vars - -- [ ] **Step 1: Draft page with docs-writer agent** - -Dispatch a docs-writer agent: - -> Write a Starlight MDX page for `docs/src/content/docs/deploy/configuration.mdx`. -> -> **Frontmatter:** title "Configuration", description "Environment variables, flags, and metastore settings for Embucket.", sidebar order 1. -> -> **Purpose:** Reference for all configuration options. -> -> **Content sections:** -> -> 1. **Configuration precedence**: flags (highest) > environment variables > .env file (loaded at startup via dotenv). -> 2. **Core runtime settings** table: -> -> | Purpose | Flag | Environment variable | Default | -> | --------------------- | ------------------------- | ----------------------- | ----------- | -> | metastore config path | `--metastore-config` | `METASTORE_CONFIG` | unset | -> | bind host | `--host` | `BUCKET_HOST` | `localhost` | -> | bind port | `--port` | `BUCKET_PORT` | `3000` | -> | result serialization | `--data-format` | `DATA_FORMAT` | `json` | -> | parser dialect | `--sql-parser-dialect` | `SQL_PARSER_DIALECT` | `snowflake` | -> | query concurrency | `--max-concurrency-level` | `MAX_CONCURRENCY_LEVEL` | `8` | -> | query timeout | `--query-timeout-secs` | `QUERY_TIMEOUT_SECS` | `1200` | -> | demo user | `--auth-demo-user` | `AUTH_DEMO_USER` | `embucket` | -> | demo password | `--auth-demo-password` | `AUTH_DEMO_PASSWORD` | `embucket` | -> | JWT signing secret | `--jwt-secret` | `JWT_SECRET` | unset | -> | tracing level | `--tracing-level` | `TRACING_LEVEL` | `info` | -> | service idle timeout | `--idle-timeout-seconds` | `IDLE_TIMEOUT_SECONDS` | `18000` | -> -> 3. **Metastore configuration**: Explain METASTORE_CONFIG pointing to a YAML file. Show minimal example (`volumes: []`). Show S3 Tables volume example: -> -> ```yaml -> volumes: -> - ident: embucket -> type: s3-tables -> database: demo -> credentials: -> credential_type: access_key -> aws-access-key-id: ACCESS_KEY -> aws-secret-access-key: SECRET_ACCESS_KEY -> arn: arn:aws:s3tables:us-east-2:123456789012:bucket/my-table-bucket -> ``` -> -> Show external Iceberg tables on S3 example: -> -> ```yaml -> volumes: -> - ident: lakehouse -> type: s3 -> region: us-east-2 -> bucket: YOUR_BUCKET_NAME -> credentials: -> credential_type: access_key -> aws-access-key-id: YOUR_ACCESS_KEY -> aws-secret-access-key: YOUR_SECRET_KEY -> -> databases: -> - ident: demo -> volume: lakehouse -> -> schemas: -> - database: demo -> schema: tpch_10 -> -> tables: -> - database: demo -> schema: tpch_10 -> table: customer -> metadata_location: s3://YOUR_BUCKET_NAME/tpch_10/customer/metadata/00001.metadata.json -> ``` -> -> 4. **Statestore settings** table: -> -> | Variable | Purpose | Default | -> | ------------------------------ | --------------------------------- | --------------------- | -> | `STATESTORE_TABLE_NAME` | DynamoDB table name | `embucket-statestore` | -> | `STATESTORE_DYNAMODB_ENDPOINT` | DynamoDB endpoint (local testing) | unset | -> | `AWS_DDB_ACCESS_KEY_ID` | DynamoDB access key | unset | -> | `AWS_DDB_SECRET_ACCESS_KEY` | DynamoDB secret key | unset | -> | `AWS_DDB_SESSION_TOKEN` | Temporary credentials token | unset | -> -> 5. **Lambda deploy-time variables** table: -> -> | Variable | Purpose | -> | --------------------- | ----------------------------------------------------------- | -> | `FUNCTION_NAME` | Override Lambda function name (default: embucket-lambda) | -> | `ENV_FILE` | Env file path (default: config/.env.lambda) | -> | `AWS_LAMBDA_ROLE_ARN` | Execution role ARN for new functions | -> | `WITH_OTEL_CONFIG` | OpenTelemetry collector config file path | -> | `FEATURES` | Cargo features, comma-separated (e.g., `state-store-query`) | -> | `LAYERS` | Additional Lambda layer ARNs | -> -> 6. **Memory and performance tuning** table: -> -> | Variable | Purpose | Default | -> | ----------------------------------- | ------------------------------- | ------- | -> | `MEM_POOL_TYPE` | Memory pool type | unset | -> | `MEM_POOL_SIZE_MB` | Memory pool size in MB | unset | -> | `DISK_POOL_SIZE_MB` | Disk pool size in MB | unset | -> | `AWS_SDK_CONNECT_TIMEOUT_SECS` | AWS SDK connection timeout | unset | -> | `AWS_SDK_OPERATION_TIMEOUT_SECS` | AWS SDK operation timeout | unset | -> | `OBJECT_STORE_TIMEOUT_SECS` | Object store operation timeout | 30 | -> | `OBJECT_STORE_CONNECT_TIMEOUT_SECS` | Object store connection timeout | 3 | -> -> 7. **Authentication defaults**: Demo credentials are `embucket`/`embucket`. Override with `AUTH_DEMO_USER` and `AUTH_DEMO_PASSWORD`. Use stronger credentials in shared environments. -> -> **Rules:** -> -> - No "Owner/Last reviewed" blockquotes -> - No "backed by the repository" or "visible in cli.rs" language -> - Import `{ Aside }` from `@astrojs/starlight/components` -> - Vale Google style - -- [ ] **Step 2: Review page with docs-reviewer agent** - -> Review `docs/src/content/docs/deploy/configuration.mdx`. Check: all 7 sections present, statestore vars included, Lambda Makefile vars included, no internal language, tables are complete and accurate. - -- [ ] **Step 3: Fix any review findings** - -- [ ] **Step 4: Verify build and commit** - -```bash -cd docs && pnpm build 2>&1 | head -50 -git add docs/src/content/docs/deploy/configuration.mdx -git commit -m "docs: add Configuration reference page" -``` - ---- - -### Task 5: Write Snowflake CLI page - -**Files:** - -- Create: `docs/src/content/docs/connect/snowflake-cli.mdx` - -**Source material:** - -- Old file (from git): `guides/snowflake-cli.mdx` - -- [ ] **Step 1: Draft page with docs-writer agent** - -Dispatch a docs-writer agent: - -> Write a Starlight MDX page for `docs/src/content/docs/connect/snowflake-cli.mdx`. -> -> **Frontmatter:** title "Snowflake CLI", description "Connect the Snowflake command-line tool to Embucket.", sidebar order 0. -> -> **Purpose:** Connect the standard Snowflake CLI to a running Embucket instance. -> -> **Content:** -> -> 1. Intro (one line): The standard Snowflake command-line tool works with Embucket through its Snowflake-compatible REST API. -> 2. Prerequisites: Python 3.8+, Snowflake CLI (`python -m pip install snowflake-cli`), a running Embucket instance (local via Docker or deployed Lambda) -> 3. Configure connection: Find config with `snow --info`. Show TOML block for local: `[connections.local]` with host=localhost, region=us-east-2, port=3000, protocol=http, database=embucket, schema=public, warehouse=em.wh, account=acc.local, user=embucket, password=embucket. Test with `snow connection test -c local`. -> 4. Run a query: `snow sql -c local -q "SELECT 1 AS ok"` with expected output. -> 5. Troubleshooting (inline): Protocol errors → set protocol=http. Auth failures → use embucket/embucket. Connection refused → check Docker is running on port 3000. No data → configure metastore or see S3 table bucket setup in deploy guide. -> -> Wherever a Function URL or Lambda deployment detail is mentioned, add an `