Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 3 additions & 89 deletions .github/actions/setup/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ runs:
id: rust-cache
uses: Swatinem/rust-cache@v2
with:
shared-key: ${{ inputs.HYPERVISOR }}-${{ inputs.CPU_VENDOR }}-${{ inputs.CACHE_SCOPE }}
shared-key: ${{ inputs.HYPERVISOR }}-${{ inputs.CPU_VENDOR }}-${{ inputs.CACHE_SCOPE }}-${{ hashFiles('tool-versions.sh') }}
workspaces: |
. -> target
crates/handler-rs -> target
Expand All @@ -111,96 +111,10 @@ runs:

echo "✅ Cargo is available: $(cargo --version)"

# Required tool versions (crates.io)
WASM_TOOLS_MIN_VERSION="1.259.0"
CARGO_HYPERLIGHT_REQUIRED_VERSION="0.1.14"
OHA_REQUIRED_VERSION="1.9.0"

# hyperlight-wasm-aot: supports either a crates.io version (e.g. "0.15.0")
# or a git revision (e.g. "734fb51"). HYPERLIGHT_GIT_URL is only used in git mode.
# IMPORTANT: hyperlight-wasm-aot bundles a specific wasmtime version internally.
# The wasmtime-aot benchmark handler loads .aot files using its own wasmtime dep,
# so the pinned wasmtime version in crates/server/Cargo.toml MUST match exactly.
HYPERLIGHT_REQUIRED_VERSION="0.15.0"
HYPERLIGHT_GIT_URL="https://github.com/hyperlight-dev/hyperlight-wasm"

COMPONENTIZE_QJS_REQUIRED_VERSION="0.3.0"

if [[ "${{ inputs.BENCH_ONLY }}" != "true" ]]; then
if command -v wasm-tools &> /dev/null; then
WASM_TOOLS_CURRENT_VERSION=$(wasm-tools --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
WASM_TOOLS_CURRENT_VERSION=${WASM_TOOLS_CURRENT_VERSION:-"unknown"}
if [[ "$WASM_TOOLS_CURRENT_VERSION" != "unknown" ]] && printf '%s\n' "$WASM_TOOLS_MIN_VERSION" "$WASM_TOOLS_CURRENT_VERSION" | sort -V -C; then
echo "wasm-tools $WASM_TOOLS_CURRENT_VERSION already available (cached)"
else
echo "wasm-tools $WASM_TOOLS_CURRENT_VERSION does not meet minimum $WASM_TOOLS_MIN_VERSION. Installing..."
cargo install wasm-tools@$WASM_TOOLS_MIN_VERSION --locked --force
fi
else
echo "Installing wasm-tools $WASM_TOOLS_MIN_VERSION..."
cargo install wasm-tools@$WASM_TOOLS_MIN_VERSION --locked
fi

# Install hyperlight-wasm-aot if not cached or wrong version.
# Can use either git revision or version (e.g., "0.15.0" or "734fb51")
# Check if this looks like a git revision (hex chars) or version
if [[ "$HYPERLIGHT_REQUIRED_VERSION" =~ ^[a-f0-9]+$ ]]; then
# Git mode - always reinstall since we can't check git revision from binary
echo "Installing hyperlight-wasm-aot from git revision $HYPERLIGHT_REQUIRED_VERSION..."
cargo install hyperlight-wasm-aot --git "$HYPERLIGHT_GIT_URL" --rev "$HYPERLIGHT_REQUIRED_VERSION" --locked --force
else
# Version mode - check if already installed
if command -v hyperlight-wasm-aot &> /dev/null; then
CURRENT_VERSION=$(hyperlight-wasm-aot --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
if [ "$CURRENT_VERSION" = "$HYPERLIGHT_REQUIRED_VERSION" ]; then
echo "hyperlight-wasm-aot $CURRENT_VERSION already available (cached)"
else
echo "Installing hyperlight-wasm-aot version $HYPERLIGHT_REQUIRED_VERSION..."
cargo install hyperlight-wasm-aot@$HYPERLIGHT_REQUIRED_VERSION --locked --force
fi
else
echo "Installing hyperlight-wasm-aot version $HYPERLIGHT_REQUIRED_VERSION..."
cargo install hyperlight-wasm-aot@$HYPERLIGHT_REQUIRED_VERSION --locked
fi
fi

# Install componentize-qjs if not cached or wrong version
if command -v componentize-qjs &> /dev/null && grep -q "componentize-qjs-cli $COMPONENTIZE_QJS_REQUIRED_VERSION" "$CARGO_HOME/.crates2.json"; then
echo "componentize-qjs $COMPONENTIZE_QJS_REQUIRED_VERSION already available (cached)"
else
echo "Installing componentize-qjs $COMPONENTIZE_QJS_REQUIRED_VERSION..."
cargo install componentize-qjs-cli@$COMPONENTIZE_QJS_REQUIRED_VERSION --locked --no-default-features --force
fi

# Install cargo-hyperlight if not cached or wrong version
if command -v cargo-hyperlight &> /dev/null; then
CARGO_HYPERLIGHT_CURRENT_VERSION=$(cargo-hyperlight --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
CARGO_HYPERLIGHT_CURRENT_VERSION=${CARGO_HYPERLIGHT_CURRENT_VERSION:-"unknown"}
if [ "$CARGO_HYPERLIGHT_CURRENT_VERSION" = "$CARGO_HYPERLIGHT_REQUIRED_VERSION" ]; then
echo "cargo-hyperlight $CARGO_HYPERLIGHT_CURRENT_VERSION already available (cached)"
else
echo "cargo-hyperlight version mismatch: found $CARGO_HYPERLIGHT_CURRENT_VERSION, need $CARGO_HYPERLIGHT_REQUIRED_VERSION. Reinstalling..."
cargo install cargo-hyperlight@$CARGO_HYPERLIGHT_REQUIRED_VERSION --locked --force
fi
else
echo "Installing cargo-hyperlight $CARGO_HYPERLIGHT_REQUIRED_VERSION..."
cargo install cargo-hyperlight@$CARGO_HYPERLIGHT_REQUIRED_VERSION --locked
fi
fi

# Install oha if not cached or wrong version
if command -v oha &> /dev/null; then
OHA_CURRENT_VERSION=$(oha --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
OHA_CURRENT_VERSION=${OHA_CURRENT_VERSION:-"unknown"}
if [ "$OHA_CURRENT_VERSION" = "$OHA_REQUIRED_VERSION" ]; then
echo "oha $OHA_CURRENT_VERSION already available (cached)"
else
echo "oha version mismatch: found $OHA_CURRENT_VERSION, need $OHA_REQUIRED_VERSION. Reinstalling..."
cargo install oha@$OHA_REQUIRED_VERSION --locked --force
fi
scripts/setup-tools.sh
else
echo "Installing oha $OHA_REQUIRED_VERSION..."
cargo install oha@$OHA_REQUIRED_VERSION --locked
scripts/setup-tools.sh --bench-only
fi

echo "Final tool status:"
Expand Down
20 changes: 20 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ updates:
directory: /
schedule:
interval: weekly
groups:
github-actions:
patterns:
- '*'
labels:
- 'benchmarks: skip'
- package-ecosystem: npm
Expand All @@ -12,6 +16,15 @@ updates:
- /js/default
schedule:
interval: weekly
groups:
npm:
patterns:
- '*'
ignore:
- dependency-name: '@bytecodealliance/*'
- dependency-name: '*componentize*'
- dependency-name: '*quickjs*'
- dependency-name: '*starlingmonkey*'
labels:
- 'benchmarks: skip'
- package-ecosystem: cargo
Expand All @@ -21,5 +34,12 @@ updates:
- /crates/hyperlight-dummy-guest
schedule:
interval: weekly
groups:
cargo:
patterns:
- '*'
ignore:
- dependency-name: 'hyperlight-*'
- dependency-name: wasmtime
labels:
- 'benchmarks: skip'
2 changes: 1 addition & 1 deletion crates/server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,6 @@ wasmtime = { version = "=36.0.14", default-features = false, features = ["cranel
hyperlight-host = { version = "0.17.0", default-features = false, features = [ "kvm", "mshv3" ]}
hyperlight-common = { version = "0.17.0", default-features = false }
hyperlight-js = { version = "0.4.0", features = ["mshv3"] }
# If updating hyperlight-wasm version, also update installing `hyperlight-wasm-aot` in action.yml
# Keep hyperlight-wasm-aot in tool-versions.sh aligned with this version.
hyperlight-wasm = { version = "0.15.0" }
hyperlight-component-macro = "0.17.0"
14 changes: 12 additions & 2 deletions docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,27 @@ Development loads published history. Use `?demo=1` for synthetic data or set

## Local Benchmarks

Requires Linux x86_64, Rust, just, access to `/dev/kvm` or `/dev/mshv`, and the
tools pinned in [CI setup](../.github/actions/setup/action.yml) on `PATH`.
Requires Linux x86_64, Rust, just, and access to `/dev/kvm` or `/dev/mshv`.
Run `just setup-tools` to install the versions pinned in
[tool-versions.sh](../tool-versions.sh). Build recipes use the tools on `PATH`.

```sh
npm ci
just setup-tools
just setup-rust-wasm-toolchain
just build-benchmark-artifacts
npm run benchmark:local -- --platform kvm
npm run dev:local
```

Set a Git revision to test an unpublished tool build. Set the matching Git URL
as well when using a fork.

```sh
COMPONENTIZE_QJS_GIT_REV=<commit> just setup-tools
HYPERLIGHT_WASM_AOT_GIT_REV=<commit> just setup-tools
```

Use `--platform mshv3` on MSHV. A full run takes at least 54 minutes per platform.
Add `--smoke` for one request per configuration or `--runtime <id>` to limit runtimes.
Raw output is in `results/local/`. Successful runs update `public/local-data/`.
Expand Down
8 changes: 8 additions & 0 deletions justfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
default:
@just --list

# Install the pinned benchmark build and load tools.
setup-tools:
scripts/setup-tools.sh

# Install the pinned load tool.
setup-bench-tools:
scripts/setup-tools.sh --bench-only

# Generate the binary WIT interface used by Rust bindings.
build-handler-wit:
node scripts/build-harness.ts wit
Expand Down
98 changes: 98 additions & 0 deletions scripts/setup-tools.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/usr/bin/env bash
set -euo pipefail

script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
source "$script_dir/../tool-versions.sh"

bench_only=false
if [[ ${1:-} == "--bench-only" ]]; then
bench_only=true
elif [[ $# -ne 0 ]]; then
echo "Usage: $0 [--bench-only]" >&2
exit 2
fi

current_version() {
local binary=$1
"$binary" --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true
}

install_exact() {
local package=$1
local binary=$2
local required=$3
shift 3

local current=""
local installed=false
if command -v "$binary" &>/dev/null; then
installed=true
current=$(current_version "$binary")
fi

if [[ $current == "$required" ]]; then
echo "$binary $current already available"
return
fi

local force=()
if [[ $installed == true ]]; then
force=(--force)
fi
echo "Installing $package $required..."
cargo install "$package@$required" --locked "${force[@]}" "$@"
}

install_git() {
local package=$1
local url=$2
local revision=$3
shift 3

echo "Installing $package from $url at $revision..."
cargo install "$package" --git "$url" --rev "$revision" --locked --force "$@"
}

install_wasm_tools() {
local current=""
local installed=false
if command -v wasm-tools &>/dev/null; then
installed=true
current=$(current_version wasm-tools)
fi

if [[ -n $current ]] && printf '%s\n' "$WASM_TOOLS_MIN_VERSION" "$current" | sort -V -C; then
echo "wasm-tools $current already available"
return
fi

local force=()
if [[ $installed == true ]]; then
force=(--force)
fi
echo "Installing wasm-tools $WASM_TOOLS_MIN_VERSION..."
cargo install "wasm-tools@$WASM_TOOLS_MIN_VERSION" --locked "${force[@]}"
}

if [[ $bench_only == false ]]; then
install_wasm_tools

if [[ -n $HYPERLIGHT_WASM_AOT_GIT_REV ]]; then
install_git hyperlight-wasm-aot "$HYPERLIGHT_WASM_AOT_GIT_URL" "$HYPERLIGHT_WASM_AOT_GIT_REV"
else
install_exact hyperlight-wasm-aot hyperlight-wasm-aot "$HYPERLIGHT_WASM_AOT_VERSION"
fi

if [[ -n $COMPONENTIZE_QJS_GIT_REV ]]; then
install_git componentize-qjs-cli "$COMPONENTIZE_QJS_GIT_URL" "$COMPONENTIZE_QJS_GIT_REV" --no-default-features
elif command -v componentize-qjs &>/dev/null && grep -q "componentize-qjs-cli $COMPONENTIZE_QJS_VERSION" "${CARGO_HOME:-$HOME/.cargo}/.crates2.json"; then
echo "componentize-qjs $COMPONENTIZE_QJS_VERSION already available"
else
echo "Installing componentize-qjs-cli $COMPONENTIZE_QJS_VERSION..."
cargo install "componentize-qjs-cli@$COMPONENTIZE_QJS_VERSION" --locked --force --no-default-features
fi

install_exact cargo-hyperlight cargo-hyperlight "$CARGO_HYPERLIGHT_VERSION"
fi

install_exact oha oha "$OHA_VERSION"
12 changes: 12 additions & 0 deletions tool-versions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
: "${WASM_TOOLS_MIN_VERSION:=1.259.0}"
: "${CARGO_HYPERLIGHT_VERSION:=0.1.14}"
: "${OHA_VERSION:=1.9.0}"

# Must use the Wasmtime version pinned by crates/server/Cargo.toml.
: "${HYPERLIGHT_WASM_AOT_VERSION:=0.15.0}"
: "${HYPERLIGHT_WASM_AOT_GIT_URL:=https://github.com/hyperlight-dev/hyperlight-wasm}"
: "${HYPERLIGHT_WASM_AOT_GIT_REV:=}"

: "${COMPONENTIZE_QJS_VERSION:=0.3.0}"
: "${COMPONENTIZE_QJS_GIT_URL:=https://github.com/andreiltd/componentize-qjs}"
: "${COMPONENTIZE_QJS_GIT_REV:=}"
Loading