From e6e814522cebe01d9c3d9cae2759ec134321c18d Mon Sep 17 00:00:00 2001 From: MK Date: Mon, 2 Mar 2026 02:06:55 -0500 Subject: [PATCH 1/7] docs: create new documentation pages Extract content from README.md into dedicated doc files: - installation.md: Homebrew, binary, and Windows install - quickstart.md: Why Forge + Get Started in 60 Seconds - memory.md: Session persistence, compactor, long-term memory - configuration.md: Full forge.yaml schema + env vars - deployment.md: Container packaging, Kubernetes, air-gap - scheduling.md: Cron config, expressions, schedule tools - dashboard.md: forge ui features and architecture - security/secrets.md: Encrypted storage, per-agent secrets - security/signing.md: Ed25519 key management and verification - security/guardrails.md: Content filtering, PII, jailbreak protection --- docs/configuration.md | 94 +++++++++++++++++ docs/dashboard.md | 113 +++++++++++++++++++++ docs/deployment.md | 92 +++++++++++++++++ docs/installation.md | 31 ++++++ docs/memory.md | 112 ++++++++++++++++++++ docs/quickstart.md | 53 ++++++++++ docs/scheduling.md | 58 +++++++++++ docs/security/guardrails.md | 66 ++++++++++++ docs/security/{SECURITY.md => overview.md} | 0 docs/security/secrets.md | 86 ++++++++++++++++ docs/security/signing.md | 49 +++++++++ 11 files changed, 754 insertions(+) create mode 100644 docs/configuration.md create mode 100644 docs/dashboard.md create mode 100644 docs/deployment.md create mode 100644 docs/installation.md create mode 100644 docs/memory.md create mode 100644 docs/quickstart.md create mode 100644 docs/scheduling.md create mode 100644 docs/security/guardrails.md rename docs/security/{SECURITY.md => overview.md} (100%) create mode 100644 docs/security/secrets.md create mode 100644 docs/security/signing.md diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..18991d3 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,94 @@ +# Configuration Reference + +> Part of [Forge Documentation](../README.md) + +All Forge agent configuration lives in `forge.yaml` at the project root. + +## Full Schema + +```yaml +agent_id: "my-agent" # Required +version: "1.0.0" # Required +framework: "forge" # forge (default), crewai, langchain +registry: "ghcr.io/org" # Container registry +entrypoint: "agent.py" # Required for crewai/langchain, omit for forge + +model: + provider: "openai" # openai, anthropic, gemini, ollama, custom + name: "gpt-4o" # Model name + fallbacks: # Fallback providers (optional) + - provider: "anthropic" + name: "claude-sonnet-4-20250514" + +tools: + - name: "web_search" + - name: "cli_execute" + config: + allowed_binaries: ["git", "curl"] + env_passthrough: ["GITHUB_TOKEN"] + +channels: + - "telegram" + - "slack" + +egress: + profile: "strict" # strict, standard, permissive + mode: "allowlist" # deny-all, allowlist, dev-open + allowed_domains: # Explicit domains + - "api.example.com" + - "*.github.com" + capabilities: # Capability bundles + - "slack" + +skills: + path: "SKILL.md" + +secrets: + providers: # Secret providers (order matters) + - "encrypted-file" # AES-256-GCM encrypted file + - "env" # Environment variables + +memory: + persistence: true # Session persistence (default: true) + sessions_dir: ".forge/sessions" + char_budget: 200000 # Context budget override + trigger_ratio: 0.6 # Compaction trigger ratio + long_term: false # Long-term memory (default: false) + memory_dir: ".forge/memory" + embedding_provider: "" # Auto-detect from LLM provider + embedding_model: "" # Provider default + vector_weight: 0.7 # Hybrid search vector weight + keyword_weight: 0.3 # Hybrid search keyword weight + decay_half_life_days: 7 # Temporal decay half-life + +schedules: # Recurring scheduled tasks (optional) + - id: "daily-report" + cron: "@daily" + task: "Generate daily status report" + skill: "" # Optional skill to invoke + channel: "telegram" # Optional channel for delivery + channel_target: "-100123456" # Destination chat/channel ID +``` + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `FORGE_MODEL_PROVIDER` | Override LLM provider | +| `FORGE_MODEL_FALLBACKS` | Fallback chain (e.g., `"anthropic:claude-sonnet-4,gemini"`) | +| `FORGE_MEMORY_PERSISTENCE` | Set `false` to disable session persistence | +| `FORGE_MEMORY_LONG_TERM` | Set `true` to enable long-term memory | +| `FORGE_EMBEDDING_PROVIDER` | Override embedding provider | +| `OPENAI_API_KEY` | OpenAI API key | +| `ANTHROPIC_API_KEY` | Anthropic API key | +| `GEMINI_API_KEY` | Google Gemini API key | +| `TAVILY_API_KEY` | Tavily web search API key | +| `PERPLEXITY_API_KEY` | Perplexity web search API key | +| `WEB_SEARCH_PROVIDER` | Force web search provider (`tavily` or `perplexity`) | +| `OPENAI_BASE_URL` | Override OpenAI base URL | +| `ANTHROPIC_BASE_URL` | Override Anthropic base URL | +| `OLLAMA_BASE_URL` | Override Ollama base URL (default: `http://localhost:11434`) | +| `FORGE_PASSPHRASE` | Passphrase for encrypted secrets file | + +--- +← [Commands](commands.md) | [Back to README](../README.md) | [Dashboard](dashboard.md) → diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 0000000..01fd307 --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,113 @@ +# Web Dashboard + +> Part of [Forge Documentation](../README.md) + +Forge includes a local web dashboard for managing agents from the browser — no CLI needed after launch. + +## Launch + +```bash +# Launch the dashboard +forge ui + +# Specify workspace and port +forge ui --dir /path/to/workspace --port 4200 + +# Launch without auto-opening browser +forge ui --no-open +``` + +Opens `http://localhost:4200` with a full-featured SPA for the complete agent lifecycle. + +## Dashboard + +The main view discovers all agents in the workspace directory and shows their status in real-time via SSE (Server-Sent Events). + +| Feature | Description | +|---------|-------------| +| Agent discovery | Auto-scans workspace for `forge.yaml` files | +| Start / Stop | Start and stop agents with one click | +| Live status | Real-time state updates (stopped, starting, running, errored) | +| Passphrase unlock | Prompts for `FORGE_PASSPHRASE` when agents have encrypted secrets | +| Auto-rescan | Detects new agents after creation | + +## Interactive Chat + +Click any running agent to open a chat interface that streams responses via the A2A protocol. + +| Feature | Description | +|---------|-------------| +| Streaming responses | Real-time token streaming with progress indicators | +| Markdown rendering | Code blocks, tables, lists rendered inline | +| Session history | Browse and resume previous conversations | +| Tool call visibility | See which tools the agent invokes during execution | + +## Create Agent Wizard + +A multi-step wizard (web equivalent of `forge init`) that walks through the full agent setup: + +| Step | What it does | +|------|-------------| +| Name | Set agent name with live slug preview | +| Provider | Select LLM provider (OpenAI, Anthropic, Gemini, Ollama, Custom) with descriptions | +| Model & Auth | Pick from provider-specific model lists; OpenAI supports API key or browser OAuth login | +| Channels | Select Slack/Telegram with inline token collection | +| Tools | Select builtin tools; web_search shows Tavily vs Perplexity provider choice with API key input | +| Skills | Browse registry skills by category with inline required/optional env var collection | +| Fallback | Select backup LLM providers with API keys for automatic failover | +| Env & Security | Add extra env vars; set passphrase for AES-256-GCM secret encryption | +| Review | Summary of all selections before creation | + +The wizard collects credentials inline at each step (matching the CLI TUI behavior) and supports all the same options: model selection, OAuth, web search providers, fallback chains, and encrypted secret storage. + +## Config Editor + +Edit `forge.yaml` for any agent with a Monaco-based YAML editor: + +| Feature | Description | +|---------|-------------| +| Syntax highlighting | YAML language support with Monaco editor | +| Live validation | Validate config against the forge schema without saving | +| Save with validation | Server-side validation before writing to disk | +| Keyboard shortcut | Cmd/Ctrl+S to save | +| Restart integration | Restart agent after config changes | +| Fallback editor | Plain textarea if Monaco fails to load | + +The Monaco editor is a tree-shaken YAML-only bundle (~615KB) built with esbuild — not the full 4MB distribution. + +## Skills Browser + +Browse the built-in skill registry with filtering and detail view: + +| Feature | Description | +|---------|-------------| +| Grid view | Skill cards showing name, description, category, tags | +| Category filter | Filter skills by category | +| Detail panel | Click a skill to view its full SKILL.md content | +| Env requirements | Shows required, one-of, and optional env vars per skill | + +## Architecture + +The dashboard is a single Go module (`forge-ui`) embedded into the `forge` binary: + +``` +forge-cli/cmd/ui.go CLI command, injects StartFunc/CreateFunc/OAuthFunc +forge-ui/ + server.go HTTP server with CORS, SPA fallback + handlers.go Dashboard API (agents, start/stop, chat, sessions) + handlers_create.go Wizard API (create, config, skills, tools, OAuth) + process.go Process manager (start/stop agent goroutines) + discovery.go Workspace scanner (finds forge.yaml files) + sse.go Server-Sent Events broker + chat.go A2A chat proxy with streaming + types.go Shared types + static/dist/ Embedded frontend (Preact + HTM, no build step) + app.js SPA with hash routing + style.css Dark theme styles + monaco/ Tree-shaken YAML editor +``` + +Key design: `forge-cli` imports `forge-ui` (not vice versa). CLI-specific logic (scaffold, config loading, OAuth flow) is injected via function callbacks, keeping `forge-ui` framework-agnostic. + +--- +← [Configuration](configuration.md) | [Back to README](../README.md) | [Deployment](deployment.md) → diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..5ca63a6 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,92 @@ +# Packaging & Deployment + +> Part of [Forge Documentation](../README.md) + +Forge agents can be packaged as container images and deployed to Docker, Kubernetes, or air-gapped environments. + +## Building Container Images + +```bash +# Build a container image (auto-detects Docker/Podman/Buildah) +forge package + +# Production build (rejects dev tools and dev-open egress) +forge package --prod + +# Build and push to registry +forge package --registry ghcr.io/myorg --push + +# Generate docker-compose with channel sidecars +forge package --with-channels + +# Export for Initializ Command platform +forge export --pretty --include-schemas +``` + +`forge package` generates a Dockerfile, Kubernetes manifests, and NetworkPolicy. Use `--prod` to strip dev tools and enforce strict egress. Use `--verify` to smoke-test the built container. + +## Production Build Checks + +Production builds (`--prod`) enforce: + +- No `dev-open` egress mode +- No dev-only tools (`local_shell`, `local_file_browser`) +- Secret provider chain must include `env` (not just `encrypted-file`) +- `.dockerignore` must exist if a Dockerfile is generated + +## Docker Compose + +```bash +forge package --with-channels +``` + +This generates a `docker-compose.yaml` with: +- An `agent` service running the A2A server +- Adapter services (e.g., `slack-adapter`, `telegram-adapter`) connecting to the agent + +## Kubernetes + +Every `forge build` generates container-ready artifacts: + +| Artifact | Purpose | +|----------|---------| +| `Dockerfile` | Container image with minimal attack surface | +| `deployment.yaml` | Kubernetes Deployment manifest | +| `service.yaml` | Kubernetes Service manifest | +| `network-policy.yaml` | NetworkPolicy restricting pod egress to allowed domains | +| `egress_allowlist.json` | Machine-readable domain allowlist | +| `checksums.json` | SHA-256 checksums + Ed25519 signature | + +## Air-Gap Deployments + +Forge can run entirely offline with local models: + +1. Use `ollama` as the LLM provider with a locally-hosted model +2. Set egress mode to `deny-all` to block all outbound traffic +3. Pre-install all binary dependencies in the container image +4. Use environment variables for secrets (no passphrase prompting needed) + +```yaml +model: + provider: ollama + name: llama3 +egress: + mode: deny-all +``` + +## Command Platform Export + +For Initializ Command integration, export the agent spec: + +```bash +# Export with embedded schemas +forge export --pretty --include-schemas + +# Simulate Command import +forge export --simulate-import +``` + +See [Command Integration](command-integration.md) for the full integration guide. + +--- +← [Dashboard](dashboard.md) | [Back to README](../README.md) | [Plugins](plugins.md) → diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..f193530 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,31 @@ +# Installation + +> Part of [Forge Documentation](../README.md) + +Forge can be installed via Homebrew, pre-built binary, or manual download on Windows. + +## macOS (Homebrew) + +```bash +brew install initializ/tap/forge +``` + +## Linux / macOS (Binary) + +```bash +curl -sSL https://github.com/initializ/forge/releases/latest/download/forge-$(uname -s)-$(uname -m).tar.gz | tar xz +sudo mv forge /usr/local/bin/ +``` + +## Windows + +Download the latest `.zip` from [GitHub Releases](https://github.com/initializ/forge/releases/latest) and add to your PATH. + +## Verify + +```bash +forge --version +``` + +--- +← [Quick Start](quickstart.md) | [Back to README](../README.md) | [Architecture](architecture.md) → diff --git a/docs/memory.md b/docs/memory.md new file mode 100644 index 0000000..a2e8762 --- /dev/null +++ b/docs/memory.md @@ -0,0 +1,112 @@ +# Memory + +> Part of [Forge Documentation](../README.md) + +Forge provides two layers of memory management: session persistence for multi-turn conversations and long-term memory for cross-session knowledge. + +## Session Persistence + +Sessions are automatically persisted to disk across requests, enabling multi-turn conversations: + +```yaml +memory: + persistence: true # default: true + sessions_dir: ".forge/sessions" +``` + +- Sessions are saved as JSON files with atomic writes (temp file + fsync + rename) +- Automatic cleanup of sessions older than 7 days at startup +- Session recovery on subsequent requests (disk snapshot supersedes task history) + +## Context Window Management + +Forge automatically manages context window usage based on model capabilities: + +| Model | Context Window | Character Budget | +|-------|---------------|-----------------| +| `gpt-4o` / `gpt-5` | 128K tokens | ~435K chars | +| `claude-sonnet` / `claude-opus` | 200K tokens | ~680K chars | +| `gemini-2.5` | 1M tokens | ~3.4M chars | +| `llama3` | 8K tokens | ~27K chars | +| `llama3.1` | 128K tokens | ~435K chars | + +When context grows too large, the **Compactor** automatically: +1. Takes the oldest 50% of messages +2. Flushes tool results and decisions to long-term memory (if enabled) +3. Summarizes via LLM (with extractive fallback) +4. Replaces old messages with the summary + +Research tool results receive special handling during compaction: they are preserved with a higher extraction limit (5000 vs 2000 characters) and tagged distinctly in long-term memory logs (e.g., `[research][tool:tavily_research]`) so research insights persist across sessions. + +```yaml +memory: + char_budget: 200000 # override auto-detection + trigger_ratio: 0.6 # compact at 60% of budget (default) +``` + +## Long-Term Memory + +Enable cross-session knowledge persistence with hybrid vector + keyword search: + +```yaml +memory: + long_term: true + memory_dir: ".forge/memory" + vector_weight: 0.7 + keyword_weight: 0.3 + decay_half_life_days: 7 +``` + +Or via environment variable: + +```bash +export FORGE_MEMORY_LONG_TERM=true +``` + +When enabled, Forge: +- Creates a `.forge/memory/` directory with a `MEMORY.md` template for curated facts +- Indexes all `.md` files into a hybrid search index (vector similarity + keyword overlap + temporal decay) +- Registers `memory_search` and `memory_get` tools for the agent to use +- Automatically flushes compacted conversation context to daily log files (`YYYY-MM-DD.md`) + +## Embedding Providers + +Embedding providers power the vector search component of long-term memory: + +| Provider | Default Model | Notes | +|----------|--------------|-------| +| `openai` | `text-embedding-3-small` | Standard OpenAI embeddings API | +| `gemini` | `text-embedding-3-small` | OpenAI-compatible endpoint | +| `ollama` | `nomic-embed-text` | Local embeddings | + +Falls back to keyword-only search if no embedding provider is available (e.g., when using Anthropic as the primary provider without a fallback). + +## Configuration + +Full memory configuration in `forge.yaml`: + +```yaml +memory: + persistence: true + sessions_dir: ".forge/sessions" + char_budget: 200000 + trigger_ratio: 0.6 + long_term: false + memory_dir: ".forge/memory" + embedding_provider: "" # Auto-detect from LLM provider + embedding_model: "" # Provider default + vector_weight: 0.7 + keyword_weight: 0.3 + decay_half_life_days: 7 +``` + +Environment variables: + +| Variable | Description | +|----------|-------------| +| `FORGE_MEMORY_PERSISTENCE` | Set `false` to disable session persistence | +| `FORGE_MEMORY_LONG_TERM` | Set `true` to enable long-term memory | +| `FORGE_EMBEDDING_PROVIDER` | Override embedding provider | + +--- +← [Runtime](runtime.md) | [Back to README](../README.md) | [Channels](channels.md) → diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..61d10be --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,53 @@ +# Quick Start + +> Part of [Forge Documentation](../README.md) + +Get a Forge agent running in under 60 seconds. + +## Why Forge? + +**Instant Agent From a Single Command** + +Write a SKILL.md. Run `forge init`. Your agent is live. + +The wizard configures your model provider, validates your API key, +connects Slack or Telegram, picks skills, and starts your agent. +Zero to running in under 60 seconds. + +**Secure by Default** + +Forge is designed for safe execution: + +* Does NOT create public tunnels +* Does NOT expose webhooks automatically +* Uses outbound-only connections (Slack Socket Mode, Telegram polling) +* Enforces outbound domain allowlists at both build-time and runtime, including subprocess HTTP via a local egress proxy +* Encrypts secrets at rest (AES-256-GCM) with per-agent isolation +* Signs build artifacts (Ed25519) for supply chain integrity +* Supports restricted network profiles with audit logging + +No accidental exposure. No hidden listeners. + +## Get Started in 60 Seconds + +```bash +# Install +curl -sSL https://github.com/initializ/forge/releases/latest/download/forge-$(uname -s)-$(uname -m).tar.gz | tar xz +sudo mv forge /usr/local/bin/ + +# Initialize a new agent (interactive wizard) +forge init my-agent + +# Run locally +cd my-agent && forge run + +# Run with Telegram +forge run --with telegram +``` + +The `forge init` wizard walks you through model provider, API key, fallback providers, tools, skills, and channel setup. Use `--non-interactive` with flags for scripted setups. + +See [Installation](installation.md) for all installation methods. + +--- +[Back to README](../README.md) | [Installation](installation.md) → diff --git a/docs/scheduling.md b/docs/scheduling.md new file mode 100644 index 0000000..23bae69 --- /dev/null +++ b/docs/scheduling.md @@ -0,0 +1,58 @@ +# Scheduling (Cron) + +> Part of [Forge Documentation](../README.md) + +Forge includes a built-in cron scheduler for recurring tasks, configurable in `forge.yaml` or created dynamically by the agent at runtime. + +## Configuration + +```yaml +schedules: + - id: daily-report + cron: "@daily" + task: "Generate and send the daily status report" + skill: "tavily-research" # optional: invoke a specific skill + channel: telegram # optional: deliver results to a channel + channel_target: "-100123456" # optional: destination chat/channel ID +``` + +## Cron Expressions + +| Format | Example | Description | +|--------|---------|-------------| +| 5-field standard | `*/15 * * * *` | Every 15 minutes | +| Aliases | `@hourly`, `@daily`, `@weekly`, `@monthly` | Common intervals | +| Intervals | `@every 5m`, `@every 1h30m` | Duration-based (minimum 1 minute) | + +## Schedule Tools + +The agent has four built-in tools for managing schedules at runtime: + +| Tool | Description | +|------|-------------| +| `schedule_set` | Create or update a recurring schedule | +| `schedule_list` | List all active and inactive schedules | +| `schedule_delete` | Remove a schedule (LLM-created only; YAML-defined cannot be deleted) | +| `schedule_history` | View execution history for scheduled tasks | + +Schedules can also be managed via the CLI: + +```bash +# List all schedules +forge schedule list +``` + +## Channel Delivery + +When a schedule includes `channel` and `channel_target`, the agent's response is automatically delivered to the specified channel after each execution. When schedules are created from channel conversations (Slack, Telegram), the channel context is automatically available so the agent can capture the delivery target. + +## Execution Details + +- **Tick interval**: 30 seconds +- **Overlap prevention**: A schedule won't fire again if its previous run is still in progress +- **Persistence**: Schedules are stored in `.forge/memory/SCHEDULES.md` and survive restarts +- **History**: The last 50 executions are recorded with status, duration, and correlation IDs +- **Audit events**: `schedule_fire`, `schedule_complete`, `schedule_skip`, `schedule_modify` + +--- +← [Guardrails](security/guardrails.md) | [Back to README](../README.md) | [Hooks](hooks.md) → diff --git a/docs/security/guardrails.md b/docs/security/guardrails.md new file mode 100644 index 0000000..c2428ff --- /dev/null +++ b/docs/security/guardrails.md @@ -0,0 +1,66 @@ +# Content Guardrails + +> Part of [Forge Documentation](../../README.md) + +The guardrail engine checks inbound and outbound messages against configurable policy rules. + +## Built-in Guardrails + +| Guardrail | Direction | Description | +|-----------|-----------|-------------| +| `content_filter` | Inbound + Outbound | Blocks messages containing configured blocked words | +| `no_pii` | Outbound | Detects email addresses, phone numbers, and SSNs via regex | +| `jailbreak_protection` | Inbound | Detects common jailbreak phrases ("ignore previous instructions", etc.) | + +## Modes + +| Mode | Behavior | +|------|----------| +| `enforce` | Blocks violating messages, returns error to caller | +| `warn` | Logs violation, allows message to pass | + +## Configuration + +Guardrails are defined in the policy scaffold, loaded from `policy-scaffold.json` or generated during `forge build`. + +Custom guardrail rules can be added to the policy scaffold: + +```json +{ + "guardrails": { + "content_filter": { + "mode": "enforce", + "blocked_words": ["password", "credit card"] + }, + "no_pii": { + "mode": "enforce" + }, + "jailbreak_protection": { + "mode": "warn" + } + } +} +``` + +## Runtime + +```bash +# Run with guardrails enforced +forge run --enforce-guardrails + +# Default: warn mode (log only) +forge run +``` + +## Audit Events + +Guardrail evaluations are logged as structured audit events: + +```json +{"ts":"2026-02-28T10:00:00Z","event":"guardrail_check","correlation_id":"a1b2c3d4","fields":{"guardrail":"no_pii","direction":"outbound","result":"blocked"}} +``` + +See [Security Overview](overview.md) for the full security architecture. + +--- +← [Build Signing](signing.md) | [Back to README](../../README.md) | [Scheduling](../scheduling.md) → diff --git a/docs/security/SECURITY.md b/docs/security/overview.md similarity index 100% rename from docs/security/SECURITY.md rename to docs/security/overview.md diff --git a/docs/security/secrets.md b/docs/security/secrets.md new file mode 100644 index 0000000..6f03b7f --- /dev/null +++ b/docs/security/secrets.md @@ -0,0 +1,86 @@ +# Secrets Management + +> Part of [Forge Documentation](../../README.md) + +Forge provides encrypted secret management with per-agent isolation and interactive passphrase prompting. + +## Encrypted Storage + +Secrets are stored in AES-256-GCM encrypted files with Argon2id key derivation. The file format is `salt(16) || nonce(12) || ciphertext`, with the plaintext being a JSON key-value map. + +```bash +# Store a secret (prompts for value securely) +forge secret set OPENAI_API_KEY + +# Store with inline value +forge secret set SLACK_BOT_TOKEN xoxb-... + +# Retrieve a secret (shows source: encrypted-file or env) +forge secret get OPENAI_API_KEY + +# List all secret keys +forge secret list + +# Delete a secret +forge secret delete OLD_KEY +``` + +## Per-Agent Secrets + +Each agent can have its own encrypted secrets file at `/.forge/secrets.enc`, separate from the global `~/.forge/secrets.enc`. Use the `--local` flag to operate on agent-local secrets: + +```bash +cd my-agent + +# Store a secret in the agent-local file +forge secret set OPENAI_API_KEY sk-agent1-key --local + +# Different agent, different key +cd ../other-agent +forge secret set OPENAI_API_KEY sk-agent2-key --local +``` + +At runtime, secrets are resolved in order: **agent-local** -> **global** -> **environment variables**. This lets you override global defaults per agent. + +## Runtime Passphrase Prompting + +When `forge run` encounters encrypted secrets and no `FORGE_PASSPHRASE` environment variable is set, it prompts interactively: + +``` +$ forge run +Enter passphrase for encrypted secrets: **** +``` + +In non-interactive environments (CI/CD), set the passphrase via environment variable: + +```bash +export FORGE_PASSPHRASE="my-passphrase" +forge run +``` + +## Smart Init Passphrase + +`forge init` detects whether `~/.forge/secrets.enc` already exists: + +- **First time**: prompts for passphrase + confirmation (new setup) +- **Subsequent**: prompts once and validates by attempting to decrypt the existing file + +## Configuration + +```yaml +secrets: + providers: + - encrypted-file # AES-256-GCM encrypted file + - env # Environment variables (fallback) +``` + +Secret files are automatically excluded from git (`.forge/` in `.gitignore`) and Docker builds (`*.enc` in `.dockerignore`). + +## File Safety + +- `.forge/` directories are automatically added to `.gitignore` +- `*.enc` files are excluded in `.dockerignore` +- Secret files never appear in container images + +--- +← [Egress Security](egress.md) | [Back to README](../../README.md) | [Build Signing](signing.md) → diff --git a/docs/security/signing.md b/docs/security/signing.md new file mode 100644 index 0000000..21d2ad7 --- /dev/null +++ b/docs/security/signing.md @@ -0,0 +1,49 @@ +# Build Signing & Verification + +> Part of [Forge Documentation](../../README.md) + +Forge supports Ed25519 signing of build artifacts for supply chain integrity. + +## Key Management + +```bash +# Generate an Ed25519 signing keypair +forge key generate +# Output: ~/.forge/signing-key.pem (private) + ~/.forge/signing-key.pub (public) + +# Generate with a custom name +forge key generate --name ci-key + +# Add a public key to the trusted keyring +forge key trust ~/.forge/signing-key.pub + +# List signing and trusted keys +forge key list +``` + +## Build Signing + +When a signing key exists at `~/.forge/signing-key.pem` (or specified via `--signing-key`), `forge build` automatically: + +1. Computes SHA-256 checksums of all generated artifacts +2. Signs the checksums with the Ed25519 private key +3. Writes `checksums.json` with checksums, signature, and key ID + +## Runtime Verification + +At runtime, `forge run` can verify build artifacts against `checksums.json`: + +- Validates SHA-256 checksums of all files +- Verifies the Ed25519 signature against trusted keys in `~/.forge/trusted-keys/` +- Verification is optional — if `checksums.json` doesn't exist, it's skipped + +## Secret Safety Stage + +The build pipeline includes a `secret-safety` stage that: + +- Blocks production builds (`--prod`) that only use `encrypted-file` without `env` provider (containers can't use encrypted files at runtime) +- Warns if `.dockerignore` is missing alongside a generated Dockerfile +- Ensures secrets never leak into container images + +--- +← [Secrets](secrets.md) | [Back to README](../../README.md) | [Guardrails](guardrails.md) → From 0f8cb9dc2dc72684eee9d9acf144891e1c62afb0 Mon Sep 17 00:00:00 2001 From: MK Date: Mon, 2 Mar 2026 02:07:12 -0500 Subject: [PATCH 2/7] docs: enrich existing pages with README content Merge README content into existing doc files: - skills.md: Add skill registry CLI, first-class tools, execution security, categories/tags, built-in skills detail, system prompt injection - tools.md: Expand builtin tools table, add web search providers, CLI execute security config, memory tools - runtime.md: Add LLM providers table, OpenAI OAuth, fallback chains, running modes (forge run vs forge serve) - hooks.md: Add OnProgress hook point, progress tracking section - channels.md: Add large response handling, Slack standalone mode - commands.md: Add forge serve, forge secret, forge key, forge schedule - architecture.md: Add at-a-glance ASCII diagram, update module tree to include forge-skills/ and forge-ui/ --- docs/architecture.md | 125 +++++++++++++++++++------- docs/channels.md | 120 ++++++++++++------------- docs/commands.md | 186 +++++++++++++++++++++++++++++++++----- docs/hooks.md | 27 +++--- docs/runtime.md | 170 ++++++++++++++++++++++++++--------- docs/skills.md | 206 ++++++++++++++++++++++++++++++++++++------- docs/tools.md | 135 ++++++++++++++++------------ 7 files changed, 714 insertions(+), 255 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index e85a7ab..3c8f692 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,24 +1,40 @@ # Architecture -## Overview +> Part of [Forge Documentation](../README.md) -Forge is a portable runtime for building and running secure AI agents from simple skill definitions. The core data flow is: +Forge is a portable runtime for building and running secure AI agents from simple skill definitions. + +## At a Glance ``` -SKILL.md → Parse → Discover tools/requirements → Compile AgentSpec → Apply security → Run LLM loop +SKILL.md --> Parse --> Discover tools/requirements --> Compile AgentSpec + | + v + Apply security policy + | + v + Run LLM agent loop + (tool calling + memory + cron) ``` -Skill definitions and `forge.yaml` configuration are compiled into a canonical `AgentSpec`, security policies are applied, and the resulting agent can be run locally, packaged into a container, or served over the A2A protocol. +1. You write a `SKILL.md` that describes what the agent can do +2. Forge parses the skill definitions and optional YAML frontmatter (binary deps, env vars) +3. The build pipeline discovers tools, resolves egress domains, and compiles an `AgentSpec` +4. Security policies (egress allowlists, capability bundles) are applied +5. Build artifacts are checksummed and optionally signed (Ed25519) +6. At runtime, encrypted secrets are decrypted and the LLM-powered tool-calling loop executes with session persistence, memory, and a cron scheduler for recurring tasks ## Module Architecture -Forge is organized as a Go workspace with three modules: +Forge is organized as a Go workspace with five modules: ``` go.work -├── forge-core/ Embeddable library -├── forge-cli/ CLI frontend -└── forge-plugins/ Channel plugin implementations +├── forge-core/ Embeddable library +├── forge-cli/ CLI frontend +├── forge-plugins/ Channel plugin implementations +├── forge-ui/ Local web dashboard +└── forge-skills/ Skill system (registry, parser, compiler) ``` ### forge-core — Library @@ -33,6 +49,14 @@ Command-line application built on top of forge-core. Includes Cobra commands, bu Messaging platform integrations that implement the `channels.ChannelPlugin` interface from forge-core. Ships Slack, Telegram, and markdown formatting plugins. +### forge-ui — Web Dashboard + +Local web dashboard for managing agents from the browser. Single Go module embedded into the `forge` binary. See [Dashboard](dashboard.md) for details. + +### forge-skills — Skill System + +Skill system including the embedded and local skill registries, SKILL.md parser, skill compiler, requirement aggregation, security analyzer, binary/env resolver, and skill signing/verification. + ## Package Map ### forge-core @@ -66,7 +90,7 @@ Messaging platform integrations that implement the `channels.ChannelPlugin` inte | Package | Responsibility | Key Types | |---------|---------------|-----------| | `cmd/forge` | Main entry point | — | -| `cmd` | CLI command implementations | `init`, `build`, `run`, `validate`, `package`, `export`, `tool`, `channel`, `skills` | +| `cmd` | CLI command implementations | `init`, `build`, `run`, `validate`, `package`, `export`, `tool`, `channel`, `skills`, `serve`, `schedule`, `secret`, `key`, `ui` | | `config` | ForgeConfig loading and YAML parsing | — | | `build` | Build pipeline stage implementations | `FrameworkAdapterStage`, `AgentSpecStage`, `ToolsStage`, `SkillsStage`, `EgressStage`, etc. | | `container` | Container image builders | `DockerBuilder`, `PodmanBuilder`, `BuildahBuilder` | @@ -91,6 +115,19 @@ Messaging platform integrations that implement the `channels.ChannelPlugin` inte | `channels/telegram` | Telegram channel adapter (polling) | | `channels/markdown` | Markdown formatting helper | +### forge-skills + +| Package | Responsibility | +|---------|---------------| +| `contract` | Skill types, registry interface, filtering | +| `local` | Embedded + local skill registries | +| `parser` | SKILL.md parser (frontmatter + body extraction) | +| `compiler` | Skill compiler (prompt generation) | +| `requirements` | Requirement aggregation and derivation | +| `analyzer` | Security audit for skills | +| `resolver` | Binary and env var resolution | +| `trust` | Skill signing and verification | + ## Key Interfaces ### `forgecore` Public API @@ -259,33 +296,55 @@ forge run → channels.Router (optional) [forge-cli/channels] ``` -## Schema Validation +## Module Directory Tree -AgentSpec JSON is validated against `schemas/agentspec.v1.0.schema.json` (JSON Schema draft-07) using the `gojsonschema` library. The schema is embedded in the binary via `go:embed` in `forge-core/schemas/`. - -Validation checks include: -- `agent_id` matches pattern `^[a-z0-9-]+$` -- `version` matches semver pattern -- Required fields: `forge_version`, `agent_id`, `version`, `name` -- Nested object schemas for runtime, tools, policy_scaffold, identity, a2a, model - -## Template System - -Templates use Go's `text/template` package and are embedded via `go:embed` in `forge-cli/templates/`. Templates are used for: - -- **Build output** — Dockerfile, Kubernetes manifests -- **Init scaffolding** — forge.yaml, agent entrypoints, tool examples, .gitignore -- **Framework wrappers** — A2A wrappers for CrewAI and LangChain - -## Runtime Architecture +``` +forge/ + forge-core/ Core library + a2a/ A2A protocol types + llm/ LLM client, fallback chains, OAuth + memory/ Long-term memory (vector + keyword search) + runtime/ Agent loop, hooks, compactor, audit logger + scheduler/ Cron scheduler (parser, tick loop, overlap prevention) + secrets/ Encrypted secret storage (AES-256-GCM + Argon2id) + security/ Egress resolver, enforcer, proxy, K8s NetworkPolicy + tools/ Tool registry, builtins, adapters, skill_tool + types/ Config types + forge-cli/ CLI application + cmd/ CLI commands (init, build, run, serve, schedule, etc.) + runtime/ Runner, skill registration, scheduler store, subprocess executor + internal/tui/ Interactive init wizard (Bubbletea) + tools/ CLI-specific tools (cli_execute, skill executor) + forge-plugins/ Channel plugins + telegram/ Telegram adapter (polling, document upload) + slack/ Slack adapter (Socket Mode, file upload) + markdown/ Markdown converter, message splitting + forge-ui/ Local web dashboard + server.go HTTP server, routing, CORS + handlers*.go REST API (agents, config, wizard, skills) + process.go Agent process manager + discovery.go Workspace scanner + sse.go Real-time event broker + chat.go A2A streaming chat proxy + static/dist/ Embedded SPA (Preact + HTM + Monaco) + forge-skills/ Skill system + contract/ Skill types, registry interface, filtering + local/ Embedded + local skill registries + parser/ SKILL.md parser (frontmatter + body extraction) + compiler/ Skill compiler (prompt generation) + requirements/ Requirement aggregation and derivation + analyzer/ Security audit for skills + resolver/ Binary and env var resolution + trust/ Skill signing and verification +``` -The local runner (`forge run`) orchestrates: +## Schema Validation -1. **Executor selection** — `LLMExecutor` (custom with LLM) lives in forge-core; `SubprocessExecutor`, `MockExecutor`, `StubExecutor` live in `forge-cli/runtime` -2. **A2A server** — JSON-RPC 2.0 HTTP server handling `tasks/send`, `tasks/get`, `tasks/cancel` (in `forge-cli/server`) -3. **Guardrail engine** — Optional inbound/outbound message checking (in `forge-core/runtime`) -4. **Channel adapters** — Optional Slack/Telegram bridges forwarding events to the A2A server (in `forge-plugins/channels`) +AgentSpec JSON is validated against `schemas/agentspec.v1.0.schema.json` (JSON Schema draft-07) using the `gojsonschema` library. The schema is embedded in the binary via `go:embed` in `forge-core/schemas/`. ## Egress Security -Egress controls operate at both build time and runtime. Build-time controls generate allowlist artifacts and Kubernetes NetworkPolicy manifests. Runtime controls include an in-process `EgressEnforcer` (Go `http.RoundTripper`) and a local `EgressProxy` for subprocess HTTP traffic. The resolver in `forge-core/security` combines explicit domains, tool-inferred domains, and capability bundles. See [security/egress.md](security/egress.md) for details. +Egress controls operate at both build time and runtime. Build-time controls generate allowlist artifacts and Kubernetes NetworkPolicy manifests. Runtime controls include an in-process `EgressEnforcer` (Go `http.RoundTripper`) and a local `EgressProxy` for subprocess HTTP traffic. See [Egress Security](security/egress.md) for details. + +--- +← [Installation](installation.md) | [Back to README](../README.md) | [Skills](skills.md) → diff --git a/docs/channels.md b/docs/channels.md index 6ec21ce..31dca79 100644 --- a/docs/channels.md +++ b/docs/channels.md @@ -1,6 +1,6 @@ # Channel Adapters -## Overview +> Part of [Forge Documentation](../README.md) Channel adapters bridge messaging platforms (Slack, Telegram) to your A2A-compliant agent. Each adapter normalizes platform-specific events into a common `ChannelEvent` format, forwards them to the agent's A2A server, and delivers responses back to the originating platform. @@ -10,6 +10,8 @@ Channel adapters bridge messaging platforms (Slack, Telegram) to your A2A-compli └──────────────── SendResponse ←────────────────────────┘ ``` +Both channels use **outbound-only connections** — no public URLs, no ngrok, no inbound webhooks. + ## Supported Channels | Channel | Adapter | Mode | Default Port | @@ -35,25 +37,46 @@ This command: 3. Adds the channel to `forge.yaml`'s `channels` list 4. Prints setup instructions +## Running with Channels + +### Alongside the Agent + +```bash +# Start agent with Slack and Telegram adapters +forge run --with slack,telegram +``` + +This starts the A2A dev server and all specified channel adapters in the same process. + +### Standalone Mode + +```bash +# Run adapter separately (requires AGENT_URL) +export AGENT_URL=http://localhost:8080 +forge channel serve slack +``` + +Standalone mode is useful for running adapters as separate services in production. Each adapter connects to the agent's A2A server via HTTP. + ## Slack App Setup Before running the Slack adapter, create and configure a Slack App: -1. **Create a Slack App** at https://api.slack.com/apps → "Create New App" → "From scratch" -2. **Enable Socket Mode** — Settings → Socket Mode → toggle **On** -3. **Generate an App-Level Token** — Basic Information → "App-Level Tokens" → "Generate Token and Scopes" → add the `connections:write` scope → copy the `xapp-...` token -4. **Enable Event Subscriptions** — Features → Event Subscriptions → toggle **On** → Subscribe to bot events: +1. **Create a Slack App** at https://api.slack.com/apps -> "Create New App" -> "From scratch" +2. **Enable Socket Mode** — Settings -> Socket Mode -> toggle **On** +3. **Generate an App-Level Token** — Basic Information -> "App-Level Tokens" -> "Generate Token and Scopes" -> add the `connections:write` scope -> copy the `xapp-...` token +4. **Enable Event Subscriptions** — Features -> Event Subscriptions -> toggle **On** -> Subscribe to bot events: - `message.channels` — messages in public channels - `message.im` — direct messages - `app_mention` — @mentions of your bot -5. **Set Bot Token Scopes** — Features → OAuth & Permissions → Bot Token Scopes → add: +5. **Set Bot Token Scopes** — Features -> OAuth & Permissions -> Bot Token Scopes -> add: - `app_mentions:read` - `chat:write` - `channels:history` - `im:history` - `files:write` (for large response file uploads) - `reactions:write` (for processing indicators) -6. **Install the App** — Settings → Install App → "Install to Workspace" → copy the `xoxb-...` Bot Token +6. **Install the App** — Settings -> Install App -> "Install to Workspace" -> copy the `xoxb-...` Bot Token 7. **Add tokens to `.env`**: ``` SLACK_APP_TOKEN=xapp-1-... @@ -61,6 +84,25 @@ Before running the Slack adapter, create and configure a Slack App: ``` 8. **Invite the bot** to any channel where you want it active: `/invite @YourBot` +### Mention-Aware Filtering + +The Slack adapter resolves the bot's own user ID at startup via `auth.test` and uses it for intelligent message filtering: + +- **Channel messages** — the bot only responds when explicitly @mentioned (e.g. `@ForgeBot what's the status?`) +- **Thread replies** — the bot responds to all messages in a thread it's participating in, unless the message @mentions a different user +- **Direct messages** — all DMs are processed +- Bot mentions are stripped from the message text before passing to the LLM, so it sees clean input + +### Processing Indicators + +When the Slack adapter receives a message: + +1. An :eyes: reaction is added immediately to acknowledge receipt +2. If the handler takes longer than 15 seconds, an interim message is posted: _"Researching, I'll post the result shortly..."_ +3. The :eyes: reaction is removed when the response is ready + +This gives users visual feedback that their message is being processed, especially for long-running research queries. + ## Configuration ### Slack (`slack-config.yaml`) @@ -94,26 +136,16 @@ Mode options: - `polling` (default) — Long-polling via `getUpdates` - `webhook` — Receives updates via HTTP webhook -## Running with Channels - -### Alongside the Agent - -```bash -# Start agent with Slack and Telegram adapters -forge run --with slack,telegram -``` +## Large Response Handling -This starts the A2A dev server and all specified channel adapters in the same process. +When an agent response exceeds 4096 characters (common with research reports), channel adapters automatically split it into a **summary message** and a **file attachment**: -### Standalone Mode +1. A brief summary (first paragraph, up to 600 characters) is sent as a regular message +2. The full report is uploaded as a downloadable Markdown file (`research-report.md`) -```bash -# Run adapter separately (requires AGENT_URL) -export AGENT_URL=http://localhost:8080 -forge channel serve slack -``` +This works on both Slack (via `files.getUploadURLExternal`) and Telegram (via `sendDocument`). If file upload fails, adapters fall back to chunked messages. Markdown is converted to platform-native formatting (Slack mrkdwn or Telegram HTML). -Standalone mode is useful for running adapters as separate services in production. +Additionally, the runtime tracks large tool outputs (>8000 characters) and attaches them as file parts in the A2A response. This ensures channel adapters receive the complete, untruncated tool output even when the LLM's text summary is truncated by output token limits. JSON tool outputs (e.g. Tavily Research/Search results) are automatically unwrapped into readable markdown before delivery. ## Docker Compose Integration @@ -132,53 +164,21 @@ Implement the `channels.ChannelPlugin` interface: ```go type ChannelPlugin interface { - // Name returns the adapter name (e.g. "slack", "telegram"). Name() string - - // Init configures the plugin from a ChannelConfig. Init(cfg ChannelConfig) error - - // Start begins listening for events and dispatching them to handler. - // It blocks until ctx is cancelled. Start(ctx context.Context, handler EventHandler) error - - // Stop gracefully shuts down the plugin. Stop() error - - // NormalizeEvent converts raw platform bytes into a ChannelEvent. NormalizeEvent(raw []byte) (*ChannelEvent, error) - - // SendResponse delivers an A2A response back to the originating platform. SendResponse(event *ChannelEvent, response *a2a.Message) error } ``` -### Key Types - -```go -// ChannelConfig holds per-adapter configuration loaded from YAML. -type ChannelConfig struct { - Adapter string `yaml:"adapter"` - WebhookPort int `yaml:"webhook_port,omitempty"` - WebhookPath string `yaml:"webhook_path,omitempty"` - Settings map[string]string `yaml:"settings,omitempty"` -} - -// ChannelEvent is the normalized representation of an inbound message. -type ChannelEvent struct { - Channel string `json:"channel"` - WorkspaceID string `json:"workspace_id"` - UserID string `json:"user_id"` - ThreadID string `json:"thread_id,omitempty"` - Message string `json:"message"` - Attachments []Attachment `json:"attachments,omitempty"` - Raw json.RawMessage `json:"raw,omitempty"` -} -``` - ### Steps -1. Create a new package under `internal/channels/yourplatform/`. +1. Create a new package under `forge-plugins/channels/yourplatform/`. 2. Implement `ChannelPlugin`. -3. Register the plugin in the channel registry (see `internal/cmd/channel.go`). +3. Register the plugin in the channel registry. 4. Add config generation in `generateChannelConfig()` and env vars in `generateEnvVars()`. + +--- +← [Memory](memory.md) | [Back to README](../README.md) | [Security Overview](security/overview.md) → diff --git a/docs/commands.md b/docs/commands.md index e5e5776..30cea98 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -1,5 +1,9 @@ # CLI Reference +> Part of [Forge Documentation](../README.md) + +Complete reference for all Forge CLI commands. + ## Global Flags | Flag | Short | Default | Description | @@ -127,6 +131,8 @@ forge run [flags] | Flag | Default | Description | |------|---------|-------------| | `--port` | `8080` | Port for the A2A dev server | +| `--host` | `""` (all interfaces) | Bind address | +| `--shutdown-timeout` | `0` (immediate) | Graceful shutdown timeout | | `--mock-tools` | `false` | Use mock runtime instead of subprocess | | `--enforce-guardrails` | `false` | Enforce guardrail violations as errors | | `--model` | | Override model name (sets `MODEL_NAME` env var) | @@ -146,12 +152,63 @@ forge run --port 9090 --mock-tools # Run with LLM provider and channels forge run --provider openai --model gpt-4 --with slack +# Container deployment +forge run --host 0.0.0.0 --shutdown-timeout 30s + # Run with guardrails enforced forge run --enforce-guardrails --env .env.production ``` --- +## `forge serve` + +Manage the agent as a background daemon process. + +``` +forge serve [start|stop|status|logs] [flags] +``` + +### Subcommands + +| Subcommand | Description | +|------------|-------------| +| `start` (default) | Start the daemon in background | +| `stop` | Send SIGTERM (10s timeout, SIGKILL fallback) | +| `status` | Show PID, listen address, health check | +| `logs` | Tail `.forge/serve.log` | + +### Flags (start) + +| Flag | Default | Description | +|------|---------|-------------| +| `--port` | `8080` | HTTP server port | +| `--host` | `127.0.0.1` | Bind address (secure default) | +| `--with` | | Channel adapters | + +### Examples + +```bash +# Start daemon (secure defaults: 127.0.0.1, 30s shutdown timeout) +forge serve + +# Start on custom port +forge serve start --port 9090 --host 0.0.0.0 + +# Stop the daemon +forge serve stop + +# Check status (PID, uptime, health) +forge serve status + +# View recent logs (last 100 lines) +forge serve logs +``` + +The daemon forks `forge run` in the background with `setsid`, writes state to `.forge/serve.json`, and redirects output to `.forge/serve.log`. + +--- + ## `forge export` Export agent spec for Command platform import. @@ -226,6 +283,18 @@ forge package --with-channels --- +## `forge schedule` + +Manage cron schedules. + +``` +forge schedule list +``` + +Lists all configured cron schedules (both YAML-defined and LLM-created). + +--- + ## `forge tool` Manage and inspect agent tools. @@ -246,16 +315,6 @@ Show tool details and input schema. forge tool describe ``` -### Examples - -```bash -# List all tools -forge tool list - -# Describe a specific tool -forge tool describe web-search -``` - --- ## `forge channel` @@ -296,22 +355,105 @@ Show configured channels from `forge.yaml`. forge channel status ``` -### Examples +--- + +## `forge secret` + +Manage encrypted secrets. ```bash -# Add Slack adapter -forge channel add slack +# Store a secret (prompts for value securely) +forge secret set OPENAI_API_KEY -# Add Telegram adapter -forge channel add telegram +# Store with inline value +forge secret set SLACK_BOT_TOKEN xoxb-... -# List available adapters -forge channel list +# Retrieve a secret (shows source) +forge secret get OPENAI_API_KEY -# Show configured channels -forge channel status +# List all secret keys +forge secret list + +# Delete a secret +forge secret delete OLD_KEY + +# Agent-local secret +forge secret set API_KEY --local +``` + +--- + +## `forge key` + +Manage Ed25519 signing keys. + +```bash +# Generate an Ed25519 signing keypair +forge key generate + +# Generate with a custom name +forge key generate --name ci-key + +# Add a public key to the trusted keyring +forge key trust ~/.forge/signing-key.pub + +# List signing and trusted keys +forge key list +``` + +--- + +## `forge skills` + +Manage agent skills. -# Run Slack adapter standalone -export AGENT_URL=http://localhost:8080 -forge channel serve slack +```bash +# Add a skill from the registry +forge skills add + +# List available skills +forge skills list + +# Filter by category +forge skills list --category sre + +# Filter by tags +forge skills list --tags kubernetes,incident-response + +# Validate skill requirements +forge skills validate + +# Audit skill security +forge skills audit --embedded + +# Sign a skill +forge skills sign + +# Generate a signing key +forge skills keygen + +# Generate trust report +forge skills trust-report ``` + +--- + +## `forge ui` + +Launch the local web dashboard. + +```bash +# Launch with defaults +forge ui + +# Specify workspace and port +forge ui --dir /path/to/workspace --port 4200 + +# Launch without auto-opening browser +forge ui --no-open +``` + +See [Dashboard](dashboard.md) for full documentation. + +--- +← [Hooks](hooks.md) | [Back to README](../README.md) | [Configuration](configuration.md) → diff --git a/docs/hooks.md b/docs/hooks.md index 5468ae0..1de19d3 100644 --- a/docs/hooks.md +++ b/docs/hooks.md @@ -1,10 +1,12 @@ # Hooks +> Part of [Forge Documentation](../README.md) + The hook system allows custom logic to run at key points in the LLM agent loop. Hooks can observe, modify context, or block execution. ## Overview -Hooks are defined in `internal/runtime/engine/hooks.go`. They fire synchronously during the agent loop and can: +Hooks fire synchronously during the agent loop and can: - **Log** interactions for debugging or auditing - **Block** execution by returning an error @@ -12,13 +14,14 @@ Hooks are defined in `internal/runtime/engine/hooks.go`. They fire synchronously ## Hook Points -| Hook Point | When It Fires | HookContext Data | +| Hook Point | When It Fires | Available Data | |-----------|---------------|------------------| -| `BeforeLLMCall` | Before each LLM API call | `Messages` | -| `AfterLLMCall` | After each LLM API call | `Messages`, `Response` | -| `BeforeToolExec` | Before each tool execution | `ToolName`, `ToolInput` | -| `AfterToolExec` | After each tool execution | `ToolName`, `ToolInput`, `ToolOutput`, `Error` | -| `OnError` | When an LLM call fails | `Error` | +| `BeforeLLMCall` | Before each LLM API call | `Messages`, `TaskID`, `CorrelationID` | +| `AfterLLMCall` | After each LLM API call | `Messages`, `Response`, `TaskID`, `CorrelationID` | +| `BeforeToolExec` | Before each tool execution | `ToolName`, `ToolInput`, `TaskID`, `CorrelationID` | +| `AfterToolExec` | After each tool execution | `ToolName`, `ToolInput`, `ToolOutput`, `Error`, `TaskID`, `CorrelationID` | +| `OnError` | When an LLM call fails | `Error`, `TaskID`, `CorrelationID` | +| `OnProgress` | During tool execution | `Phase`, `ToolName`, `StatusMessage` | ## HookContext @@ -70,6 +73,10 @@ hooks.Register(engine.BeforeToolExec, func(ctx context.Context, hctx *engine.Hoo }) ``` +## Progress Tracking + +The runner automatically registers progress hooks that emit real-time status updates during tool execution. Progress events include the tool name, phase (`tool_start` / `tool_end`), and a human-readable status message. These events are streamed to clients via SSE when using the A2A HTTP server, enabling live progress indicators in web and chat UIs. + ## Error Handling - Hooks fire **in registration order** for each hook point @@ -94,7 +101,5 @@ exec := engine.NewLLMExecutor(engine.LLMExecutorConfig{ If no `HookRegistry` is provided, an empty one is created automatically. -## Related Files - -- `internal/runtime/engine/hooks.go` — Hook types, registry, and firing logic -- `internal/runtime/engine/loop.go` — Hook integration in the agent loop +--- +← [Scheduling](scheduling.md) | [Back to README](../README.md) | [Commands](commands.md) → diff --git a/docs/runtime.md b/docs/runtime.md index 77e681e..d08fd13 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -1,10 +1,12 @@ # LLM Runtime Engine +> Part of [Forge Documentation](../README.md) + The runtime engine powers `forge run` — executing agent tasks via LLM providers with tool calling, conversation memory, and lifecycle hooks. ## Agent Loop -The core agent loop is implemented in `internal/runtime/engine/loop.go`. It follows a simple pattern: +The core agent loop follows a simple pattern: 1. **Initialize memory** with the system prompt and task history 2. **Append** the user message @@ -19,6 +21,78 @@ User message → Memory → LLM → tool_calls? → Execute tools → LLM → .. The loop terminates when `FinishReason == "stop"` or `len(ToolCalls) == 0`. +## LLM Providers + +Forge supports multiple LLM providers with automatic fallback: + +| Provider | Default Model | Auth | +|----------|--------------|------| +| `openai` | `gpt-5.2-2025-12-11` | API key or OAuth | +| `anthropic` | `claude-sonnet-4-20250514` | API key | +| `gemini` | `gemini-2.5-flash` | API key | +| `ollama` | `llama3` | None (local) | +| Custom | Configurable | API key | + +### Configuration + +```yaml +model: + provider: openai + name: gpt-4o +``` + +Or override with environment variables: + +```bash +export FORGE_MODEL_PROVIDER=anthropic +export ANTHROPIC_API_KEY=sk-ant-... +forge run +``` + +Provider is auto-detected from available API keys if not explicitly set. Provider configuration is resolved via `ResolveModelConfig()` in priority order: + +1. **CLI flag** `--provider` (highest priority) +2. **Environment variables**: `FORGE_MODEL_PROVIDER`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY` +3. **forge.yaml** `model` section (lowest priority) + +### OpenAI OAuth + +For OpenAI, Forge supports browser-based OAuth login (matching the Codex CLI flow) as an alternative to API keys: + +```bash +forge init my-agent +# Select "OpenAI" -> "Login with browser (OAuth)" +# Browser opens for authentication +``` + +OAuth tokens are stored in `~/.forge/credentials/openai.json` and automatically refreshed. + +### Fallback Chains + +Configure fallback providers for automatic failover when the primary provider is unavailable: + +```yaml +model: + provider: openai + name: gpt-4o + fallbacks: + - provider: anthropic + name: claude-sonnet-4-20250514 + - provider: gemini +``` + +Or via environment variable: + +```bash +export FORGE_MODEL_FALLBACKS="anthropic:claude-sonnet-4-20250514,gemini:gemini-2.5-flash" +``` + +Fallback behavior: +- **Retriable errors** (rate limits, overloaded, timeouts) try the next provider +- **Non-retriable errors** (auth, billing, bad format) abort immediately +- Per-provider exponential backoff cooldowns prevent thundering herd +- Fallbacks are also auto-detected from available API keys when not explicitly configured + ## Executor Types The runtime supports multiple executor implementations: @@ -29,61 +103,75 @@ The runtime supports multiple executor implementations: | `SubprocessExecutor` | Framework agents (CrewAI, LangChain) running as subprocesses | | `StubExecutor` | Returns canned responses for testing | -Executor selection happens in `internal/runtime/runner.go` based on framework type and configuration. +Executor selection happens in `runner.go` based on framework type and configuration. -## Provider Configuration +## Running Modes -Provider configuration is resolved in `internal/runtime/engine/config.go` via `ResolveModelConfig()`. Sources are checked in priority order: +### `forge run` — Foreground Server -1. **CLI flag** `--provider` (highest priority) -2. **Environment variables**: `FORGE_MODEL_PROVIDER`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `LLM_API_KEY` -3. **forge.yaml** `model` section (lowest priority) +Run the agent as a foreground HTTP server. Used for development and container deployments. -If no provider is explicitly set, the system auto-detects from available API keys. +```bash +# Development (all interfaces, immediate shutdown) +forge run --with slack --port 8080 -### Supported Providers +# Container deployment +forge run --host 0.0.0.0 --shutdown-timeout 30s +``` -| Provider | Default Model | Base URL Override | -|----------|--------------|-------------------| -| `openai` | `gpt-4o` | `OPENAI_BASE_URL` | -| `anthropic` | `claude-sonnet-4-20250514` | `ANTHROPIC_BASE_URL` | -| `ollama` | `llama3` | `OLLAMA_BASE_URL` | +| Flag | Default | Description | +|------|---------|-------------| +| `--port` | `8080` | HTTP server port | +| `--host` | `""` (all interfaces) | Bind address | +| `--shutdown-timeout` | `0` (immediate) | Graceful shutdown timeout | +| `--with` | — | Channel adapters (e.g. `slack,telegram`) | +| `--mock-tools` | `false` | Use mock executor for testing | +| `--model` | — | Override model name | +| `--provider` | — | Override LLM provider | +| `--env` | `.env` | Path to env file | +| `--enforce-guardrails` | `false` | Enforce guardrail violations as errors | -All providers implement the `llm.Client` interface defined in `internal/runtime/llm/client.go`: +### `forge serve` — Background Daemon -```go -type Client interface { - Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) - ChatStream(ctx context.Context, req *ChatRequest) (<-chan StreamDelta, error) - ModelID() string -} -``` +Manage the agent as a background daemon process with PID/log management. -## Conversation Memory +```bash +# Start daemon (secure defaults: 127.0.0.1, 30s shutdown timeout) +forge serve -Memory management is handled by `internal/runtime/engine/memory.go`. Key behaviors: +# Start on custom port +forge serve start --port 9090 --host 0.0.0.0 -- **System prompt** is always prepended to the message list (never trimmed) -- **Character budget** defaults to 32,000 characters (~8,000 tokens) -- When over budget, **oldest messages are trimmed first** -- The **most recent message is never trimmed** -- Memory is per-task (created fresh for each `Execute` call) -- Thread-safe via `sync.Mutex` +# Stop the daemon +forge serve stop -## Streaming +# Check status (PID, uptime, health) +forge serve status -The current implementation (v1) runs the full tool-calling loop non-streaming. `ExecuteStream` calls `Execute` internally and emits the final response as a single message on a channel. True word-by-word streaming during tool loops is planned for v2. +# View recent logs (last 100 lines) +forge serve logs +``` + +| Subcommand | Description | +|------------|-------------| +| `start` (default) | Start the daemon in background | +| `stop` | Send SIGTERM (10s timeout, SIGKILL fallback) | +| `status` | Show PID, listen address, health check | +| `logs` | Tail `.forge/serve.log` | + +The daemon forks `forge run` in the background with `setsid`, writes state to `.forge/serve.json`, and redirects output to `.forge/serve.log`. Passphrase prompting for encrypted secrets happens in the parent process (which has TTY access) before forking. + +## Conversation Memory + +For details on session persistence, context window management, compaction, and long-term memory, see [Memory](memory.md). ## Hooks -The engine fires hooks at key points in the loop. See [docs/hooks.md](hooks.md) for details. +The engine fires hooks at key points in the loop. See [Hooks](hooks.md) for details. -## Related Files +## Streaming + +The current implementation (v1) runs the full tool-calling loop non-streaming. `ExecuteStream` calls `Execute` internally and emits the final response as a single message on a channel. True word-by-word streaming during tool loops is planned for v2. -- `internal/runtime/engine/loop.go` — Agent loop implementation -- `internal/runtime/engine/memory.go` — Conversation memory -- `internal/runtime/engine/config.go` — Provider configuration resolution -- `internal/runtime/engine/hooks.go` — Hook system -- `internal/runtime/llm/client.go` — LLM client interface -- `internal/runtime/llm/types.go` — Canonical chat types -- `internal/runtime/llm/providers/` — Provider implementations +--- +← [Tools](tools.md) | [Back to README](../README.md) | [Memory](memory.md) → diff --git a/docs/skills.md b/docs/skills.md index b4dcd74..32cc2a4 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -1,5 +1,7 @@ # Skills +> Part of [Forge Documentation](../README.md) + Skills are a progressive disclosure mechanism for defining agent capabilities in a structured, human-readable format. They compile into container artifacts during `forge build`. ## Overview @@ -10,10 +12,6 @@ Skills bridge the gap between high-level capability descriptions and the tool-ca Skills are defined in a Markdown file (default: `SKILL.md`). The file supports optional YAML frontmatter and two body formats. -### YAML Frontmatter - -Skills can declare metadata and requirements in a YAML frontmatter block delimited by `---`: - ```markdown --- name: weather @@ -29,54 +27,202 @@ metadata: optional: [] --- ## Tool: weather_current + Get current weather for a location. + +**Input:** location (string) - City name or coordinates +**Output:** Current temperature, conditions, humidity, and wind speed + +## Tool: weather_forecast + +Get weather forecast for a location. + +**Input:** location (string), days (integer: 1-7) +**Output:** Daily forecast with high/low temperatures and conditions ``` +Each `## Tool:` heading defines a tool the agent can call. The frontmatter declares binary dependencies and environment variable requirements. Skills compile into JSON artifacts and prompt text during `forge build`. + +### YAML Frontmatter + The `metadata.forge.requires` block declares: - **`bins`** — Binary dependencies that must be in `$PATH` at runtime - **`env.required`** — Environment variables that must be set - **`env.one_of`** — At least one of these environment variables must be set - **`env.optional`** — Optional environment variables for extended functionality -Frontmatter is parsed by `ParseWithMetadata()` in `forge-core/skills/parser.go` and feeds into the compilation pipeline. The `SkillMetadata` and `SkillRequirements` types are defined in `forge-core/skills/types.go`. +Frontmatter is parsed by `ParseWithMetadata()` in `forge-core/skills/parser.go` and feeds into the compilation pipeline. -### Tool Heading Format (recommended) +### Legacy List Format ```markdown -## Tool: web_search -Search the web for current information and return relevant results. +# Agent Skills + +- translate +- summarize +- classify +``` + +Single-word list items (no spaces, max 64 characters) create name-only skill entries. This format is simpler but provides less metadata. -**Input:** query: string, max_results: int -**Output:** results: []string +## Skill Registry -## Tool: summarize -Summarize long text into a concise paragraph. +Forge ships with a built-in skill registry. Add skills to your project with a single command: -**Input:** text: string, max_length: int -**Output:** summary: string +```bash +# Add a skill from the registry +forge skills add tavily-research + +# Validate skill requirements +forge skills validate + +# Audit skill security +forge skills audit --embedded ``` -Each `## Tool:` heading starts a new skill entry. Paragraph text becomes the description. `**Input:**` and `**Output:**` lines set the input/output specifications. +`forge skills add` copies the skill's SKILL.md and any associated scripts into your project's `skills/` directory. It validates binary and environment requirements, checks for existing values in your environment, `.env` file, and encrypted secrets, and prompts only for truly missing values with a suggestion to use `forge secrets set` for sensitive keys. -### Legacy List Format +## Skills as First-Class Tools + +Script-backed skills are automatically registered as **first-class LLM tools** at runtime. When a skill has scripts in `skills/scripts/`, Forge: + +1. Parses the skill's SKILL.md for tool definitions, descriptions, and input schemas +2. Creates a named tool for each `## Tool:` entry (e.g., `tavily_research` becomes a tool the LLM can call directly) +3. Executes the skill's shell script with JSON input when the LLM invokes it + +This means the LLM sees skill tools alongside builtins like `web_search` and `http_request` — no generic `cli_execute` indirection needed. + +For skills **without** scripts (binary-backed skills like `k8s-incident-triage`), Forge injects the full skill instructions into the system prompt. The complete SKILL.md body — including triage steps, detection heuristics, output structure, and safety constraints — is included inline so the LLM follows the skill protocol without needing an extra tool call. Skills are invoked via `cli_execute` with the declared binary dependencies. + +``` +┌─────────────────────────────────────────────────┐ +│ LLM Tool Registry │ +├─────────────────┬───────────────────────────────┤ +│ Builtins │ web_search, http_request │ +│ Skill Tools │ tavily_research, ... │ ← auto-registered from scripts +│ read_skill │ load any SKILL.md on demand │ +│ cli_execute │ run approved binaries │ +├─────────────────┴───────────────────────────────┤ +│ System Prompt: full skill instructions inline │ ← binary-backed skills +└─────────────────────────────────────────────────┘ +``` + +## Skill Execution Security + +Skill scripts run in a restricted environment via `SkillCommandExecutor`: + +- **Isolated environment**: Only `PATH`, `HOME`, and explicitly declared env vars are passed through +- **Configurable timeout**: Each skill declares a `timeout_hint` in its YAML frontmatter (e.g., 300s for research) +- **No shell execution**: Scripts run via `bash