From 8f271eb35ba846a725bac2474587993fb1212e57 Mon Sep 17 00:00:00 2001 From: Matt Apperson Date: Mon, 27 Oct 2025 21:32:02 -0400 Subject: [PATCH 1/3] WIP basic tests (yes they are failing) --- package.json | 5 +- tests/e2e/README.md | 69 +++++++++++++++ tests/e2e/chat.test.ts | 186 +++++++++++++++++++++++++++++++++++++++ tests/e2e/models.test.ts | 59 +++++++++++++ vitest.config.ts | 11 +++ 5 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/chat.test.ts create mode 100644 tests/e2e/models.test.ts create mode 100644 vitest.config.ts diff --git a/package.json b/package.json index e4bf50ae..c255e413 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,10 @@ "scripts": { "lint": "eslint --cache --max-warnings=0 src", "build": "tsc", - "prepublishOnly": "npm run build" + "prepublishOnly": "npm run build", + "test": "vitest run", + "test:e2e": "vitest run tests/e2e", + "test:watch": "vitest" }, "peerDependencies": { "@tanstack/react-query": "^5", diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..3876d106 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,69 @@ +# E2E Tests + +This directory contains end-to-end tests for the OpenRouter SDK. + +## Prerequisites + +1. Install dependencies: + ```bash + npm install + ``` + +2. Set up your OpenRouter API key: + ```bash + export OPENROUTER_API_KEY=your_api_key_here + ``` + + Or create a `.env` file in the project root: + ``` + OPENROUTER_API_KEY=your_api_key_here + ``` + +## Running Tests + +Run all tests: +```bash +npm test +``` + +Run only e2e tests: +```bash +npm run test:e2e +``` + +Run tests in watch mode: +```bash +npm run test:watch +``` + +## Test Coverage + +The e2e test suite includes: + +### Models Tests (`models.test.ts`) +- Fetching the list of available models +- Validating model properties +- Filtering models by category +- Getting the total count of models + +### Chat Tests (`chat.test.ts`) +- **Non-streaming mode:** + - Sending chat requests and receiving responses + - Multi-turn conversations + - Token limit enforcement + +- **Streaming mode:** + - Streaming chat responses + - Progressive content delivery + - Finish reason detection + +### Beta Responses Tests (`responses.test.ts`) +- Testing the beta responses endpoint +- Note: This endpoint is in alpha/beta and may require updates + +## Notes + +- Tests make real API calls to OpenRouter, so you need a valid API key +- Tests may consume API credits +- Some tests use the `openai/gpt-3.5-turbo` model by default +- The beta responses endpoint has limited test coverage as it's still in development diff --git a/tests/e2e/chat.test.ts b/tests/e2e/chat.test.ts new file mode 100644 index 00000000..ab06bd99 --- /dev/null +++ b/tests/e2e/chat.test.ts @@ -0,0 +1,186 @@ +import { beforeAll, describe, expect, it } from "vitest"; +import { OpenRouter } from "../../src/sdk/sdk.js"; + +describe("Chat E2E Tests", () => { + let client: OpenRouter; + + beforeAll(() => { + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey) { + throw new Error( + "OPENROUTER_API_KEY environment variable is required for e2e tests" + ); + } + + client = new OpenRouter({ + apiKey, + }); + }); + + describe("chat.send() - Non-streaming", () => { + it("should successfully send a chat request and get a response", async () => { + const response = await client.chat.send({ + model: "meta-llama/llama-3.2-1b-instruct", + messages: [ + { + role: "user", + content: "Say 'Hello, World!' and nothing else.", + }, + ], + stream: false, + }); + + expect(response).toBeDefined(); + + + expect(Array.isArray(response.choices)).toBe(true); + expect(response.choices.length).toBeGreaterThan(0); + + const firstChoice = response.choices[0]; + expect(firstChoice).toBeDefined(); + expect(firstChoice?.message).toBeDefined(); + expect(firstChoice?.message?.content).toBeDefined(); + expect(typeof firstChoice?.message?.content).toBe("string"); + + // Verify it has usage information + expect(response.usage).toBeDefined(); + expect(response.usage?.totalTokens).toBeGreaterThan(0); + + }); + + it("should handle multi-turn conversations", async () => { + const response = await client.chat.send({ + model: "meta-llama/llama-3.2-1b-instruct", + messages: [ + { + role: "user", + content: "My name is Alice.", + }, + { + role: "assistant", + content: "Hello Alice! How can I help you today?", + }, + { + role: "user", + content: "What is my name?", + }, + ], + stream: false, + }); + + expect(response).toBeDefined(); + + const content = typeof response.choices[0]?.message?.content === "string" ? response.choices[0]?.message?.content?.toLowerCase() : response.choices[0]?.message?.content?.map((item) => item.type === "text" ? item.text : "").join("").toLowerCase(); + expect(content).toBeDefined(); + expect(content).toContain("alice"); + + }); + + it("should respect max_tokens parameter", async () => { + const response = await client.chat.send({ + model: "meta-llama/llama-3.2-1b-instruct", + messages: [ + { + role: "user", + content: "Write a long story about a cat.", + }, + ], + maxTokens: 10, + stream: false, + }); + + expect(response).toBeDefined(); + + expect(response.usage?.completionTokens).toBeLessThanOrEqual(10); + + }); + }); + + describe("chat.send() - Streaming", () => { + it("should successfully stream chat responses", async () => { + const response = await client.chat.send({ + model: "meta-llama/llama-3.2-1b-instruct", + messages: [ + { + role: "user", + content: "Count from 1 to 5.", + }, + ], + stream: true, + }); + + expect(response).toBeDefined(); + + const chunks: any[] = []; + + for await (const chunk of response) { + expect(chunk).toBeDefined(); + chunks.push(chunk); + } + + expect(chunks.length).toBeGreaterThan(0); + + // Verify chunks have expected structure + const firstChunk = chunks[0]; + expect(firstChunk?.choices).toBeDefined(); + expect(Array.isArray(firstChunk?.choices)).toBe(true); + + }); + + it("should stream complete content progressively", async () => { + const response = await client.chat.send({ + model: "meta-llama/llama-3.2-1b-instruct", + messages: [ + { + role: "user", + content: "Say 'test'.", + }, + ], + stream: true, + }); + + expect(response).toBeDefined(); + + let fullContent = ""; + let chunkCount = 0; + + for await (const chunk of response) { + chunkCount++; + const delta = chunk.choices?.[0]?.delta; + if (delta?.content) { + fullContent += delta.content; + } + } + + expect(chunkCount).toBeGreaterThan(0); + expect(fullContent.length).toBeGreaterThan(0); + } + + it("should include finish_reason in final chunk", async () => { + const response = await client.chat.send({ + model: "meta-llama/llama-3.2-1b-instruct", + messages: [ + { + role: "user", + content: "Say 'done'.", + }, + ], + stream: true, + }); + + expect(response).toBeDefined(); + + let foundFinishReason = false; + + for await (const chunk of response) { + const finishReason = chunk.choices?.[0]?.finishReason; + if (finishReason) { + foundFinishReason = true; + expect(typeof finishReason).toBe("string"); + } + } + + expect(foundFinishReason).toBe(true); + } + }); +}); diff --git a/tests/e2e/models.test.ts b/tests/e2e/models.test.ts new file mode 100644 index 00000000..08c6f65c --- /dev/null +++ b/tests/e2e/models.test.ts @@ -0,0 +1,59 @@ +import { beforeAll, describe, expect, it } from "vitest"; +import { OpenRouter } from "../../src/sdk/sdk.js"; + +describe("Models E2E Tests", () => { + let client: OpenRouter; + + beforeAll(() => { + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey) { + throw new Error( + "OPENROUTER_API_KEY environment variable is required for e2e tests" + ); + } + + client = new OpenRouter({ + apiKey, + }); + }); + + describe("models.list()", () => { + it("should successfully fetch models list", async () => { + const response = await client.models.list(); + + expect(response).toBeDefined(); + expect(Array.isArray(response)).toBe(true); + expect(response.length).toBeGreaterThan(0); + }); + + it("should return models with expected properties", async () => { + const response = await client.models.list(); + + const firstModel = response?.[0]; + expect(firstModel).toBeDefined(); + expect(firstModel?.id).toBeDefined(); + expect(typeof firstModel?.id).toBe("string"); + expect(firstModel?.name).toBeDefined(); + }); + + it("should support filtering by category", async () => { + const response = await client.models.list({ + category: "text", + }); + + expect(response).toBeDefined(); + expect(Array.isArray(response)).toBe(true); + }); + }); + + describe("models.count()", () => { + it("should successfully fetch models count", async () => { + const response = await client.models.count(); + + expect(response).toBeDefined(); + expect(response.count).toBeDefined(); + expect(typeof response.count).toBe("number"); + expect(response.count).toBeGreaterThan(0); + }); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 00000000..c612fb10 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + testMatch: ["**/*.test.ts"], + hookTimeout: 30000, + testTimeout: 30000, + }, +}); From 7eb198a1f7c17855d8e2ab683390db5896d39c74 Mon Sep 17 00:00:00 2001 From: Matt Apperson Date: Mon, 27 Oct 2025 21:37:20 -0400 Subject: [PATCH 2/3] ci: add test running to speakeasy workflow --- .github/workflows/speakeasy_run_on_pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/speakeasy_run_on_pr.yaml b/.github/workflows/speakeasy_run_on_pr.yaml index e1ff6400..be703e29 100644 --- a/.github/workflows/speakeasy_run_on_pr.yaml +++ b/.github/workflows/speakeasy_run_on_pr.yaml @@ -61,6 +61,9 @@ jobs: working-directory: examples/nextjs-example run: npx tsc --noEmit + - name: Run tests + run: npm test + - name: Commit changes run: | git config --global user.name 'github-actions[bot]' From 0043f72cd9c2496ee4876658130a885b0ef47c54 Mon Sep 17 00:00:00 2001 From: Matt Apperson Date: Mon, 27 Oct 2025 21:43:02 -0400 Subject: [PATCH 3/3] ci: refactor validation checks into reusable workflow - Create composite action for SDK validation steps (build, typecheck, tests) - Update speakeasy workflow to use composite action - Add new PR validation workflow that runs on non-OpenAPI changes - Create reusable validation-checks workflow for shared logic --- .github/actions/validate-sdk/action.yaml | 47 ++++++++++++++++++++++ .github/workflows/pr-validation.yaml | 10 +++++ .github/workflows/speakeasy_run_on_pr.yaml | 35 +--------------- .github/workflows/validation-checks.yaml | 14 +++++++ 4 files changed, 73 insertions(+), 33 deletions(-) create mode 100644 .github/actions/validate-sdk/action.yaml create mode 100644 .github/workflows/pr-validation.yaml create mode 100644 .github/workflows/validation-checks.yaml diff --git a/.github/actions/validate-sdk/action.yaml b/.github/actions/validate-sdk/action.yaml new file mode 100644 index 00000000..bcfc7598 --- /dev/null +++ b/.github/actions/validate-sdk/action.yaml @@ -0,0 +1,47 @@ +name: 'Validate SDK' +description: 'Run build, typecheck, and tests for the SDK' + +runs: + using: 'composite' + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + + - name: Install dependencies + shell: bash + run: npm install + + - name: Build SDK + shell: bash + run: npm run build + + - name: Typecheck tests directory + shell: bash + run: npx tsc --noEmit --skipLibCheck --esModuleInterop --moduleResolution node --module esnext --target es2020 'tests/**/*.ts' + + - name: Install examples dependencies + shell: bash + working-directory: examples + run: npm install + + - name: Typecheck examples root + shell: bash + working-directory: examples + run: npx tsc --noEmit --skipLibCheck --esModuleInterop --moduleResolution node --module esnext --target es2020 '*.ts' + + - name: Install nextjs-example dependencies + shell: bash + working-directory: examples/nextjs-example + run: npm install + + - name: Typecheck nextjs-example + shell: bash + working-directory: examples/nextjs-example + run: npx tsc --noEmit + + - name: Run tests + shell: bash + run: npm test diff --git a/.github/workflows/pr-validation.yaml b/.github/workflows/pr-validation.yaml new file mode 100644 index 00000000..892f563e --- /dev/null +++ b/.github/workflows/pr-validation.yaml @@ -0,0 +1,10 @@ +name: PR Validation + +on: + pull_request: + paths-ignore: + - .speakeasy/in.openapi.yaml + +jobs: + validate: + uses: ./.github/workflows/validation-checks.yaml diff --git a/.github/workflows/speakeasy_run_on_pr.yaml b/.github/workflows/speakeasy_run_on_pr.yaml index be703e29..a6cd0bad 100644 --- a/.github/workflows/speakeasy_run_on_pr.yaml +++ b/.github/workflows/speakeasy_run_on_pr.yaml @@ -30,39 +30,8 @@ jobs: - name: Run Speakeasy run: speakeasy run - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: 'npm' - - - name: Install dependencies - run: npm install - - - name: Build SDK - run: npm run build - - - name: Typecheck tests directory - run: npx tsc --noEmit --skipLibCheck --esModuleInterop --moduleResolution node --module esnext --target es2020 'tests/**/*.ts' - - - name: Install examples dependencies - working-directory: examples - run: npm install - - - name: Typecheck examples root - working-directory: examples - run: npx tsc --noEmit --skipLibCheck --esModuleInterop --moduleResolution node --module esnext --target es2020 '*.ts' - - - name: Install nextjs-example dependencies - working-directory: examples/nextjs-example - run: npm install - - - name: Typecheck nextjs-example - working-directory: examples/nextjs-example - run: npx tsc --noEmit - - - name: Run tests - run: npm test + - name: Validate SDK + uses: ./.github/actions/validate-sdk - name: Commit changes run: | diff --git a/.github/workflows/validation-checks.yaml b/.github/workflows/validation-checks.yaml new file mode 100644 index 00000000..8f6edd8b --- /dev/null +++ b/.github/workflows/validation-checks.yaml @@ -0,0 +1,14 @@ +name: Validation Checks + +on: + workflow_call: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Validate SDK + uses: ./.github/actions/validate-sdk