Skip to content

Commit d2d50d7

Browse files
yossiovadiayossiovadia
committed
Merge remote-tracking branch 'upstream/main' into fix/pii-lora-auto-detect
2 parents 3aca060 + ee6e87e commit d2d50d7

File tree

18 files changed

+2468
-84
lines changed

18 files changed

+2468
-84
lines changed

‎.github/workflows/integration-test-k8s.yml‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ jobs:
1313
integration-test:
1414
runs-on: ubuntu-latest
1515
timeout-minutes: 60
16+
strategy:
17+
fail-fast: false # Continue testing other profiles even if one fails
18+
matrix:
19+
profile: [ai-gateway, aibrix]
1620

1721
steps:
1822
- name: Check out the repo
@@ -61,11 +65,11 @@ jobs:
6165
run: |
6266
make build-e2e
6367
64-
- name: Run Integration E2E tests
68+
- name: Run Integration E2E tests (${{ matrix.profile }})
6569
id: e2e-test
6670
run: |
6771
set +e # Don't exit on error, we want to capture the result
68-
make e2e-test E2E_PROFILE=ai-gateway E2E_VERBOSE=true E2E_KEEP_CLUSTER=false
72+
make e2e-test E2E_PROFILE=${{ matrix.profile }} E2E_VERBOSE=true E2E_KEEP_CLUSTER=false
6973
TEST_EXIT_CODE=$?
7074
echo "test_exit_code=${TEST_EXIT_CODE}" >> $GITHUB_OUTPUT
7175
exit ${TEST_EXIT_CODE}
@@ -74,7 +78,7 @@ jobs:
7478
if: always()
7579
uses: actions/upload-artifact@v4
7680
with:
77-
name: test-reports
81+
name: test-reports-${{ matrix.profile }}
7882
path: |
7983
test-report.json
8084
test-report.md
@@ -123,25 +127,26 @@ jobs:
123127
fi
124128
125129
# Add additional context
126-
cat >> $GITHUB_STEP_SUMMARY << 'EOF'
130+
cat >> $GITHUB_STEP_SUMMARY << EOF
127131
128132
---
129133
130134
### 📚 Additional Resources
131135
136+
- **Profile:** \`${{ matrix.profile }}\`
132137
- **Trigger:** ${{ github.event_name }}
133-
- **Branch:** `${{ github.ref_name }}`
134-
- **Commit:** `${{ github.sha }}`
138+
- **Branch:** \`${{ github.ref_name }}\`
139+
- **Commit:** \`${{ github.sha }}\`
135140
- **Workflow Run:** [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
136141
- [E2E Test Framework Documentation](https://github.com/${{ github.repository }}/tree/main/e2e)
137-
- [AI Gateway Profile](https://github.com/${{ github.repository }}/tree/main/e2e/profiles/ai-gateway)
142+
- [${{ matrix.profile }} Profile](https://github.com/${{ github.repository }}/tree/main/e2e/profiles/${{ matrix.profile }})
138143
139144
### 📦 Artifacts
140145
141146
- **test-report.json** - Detailed test results in JSON format
142147
- **test-report.md** - Human-readable test report
143148
- **semantic-router-logs.txt** - Complete semantic-router pod logs
144-
- All artifacts are retained for 30 days
149+
- All artifacts are retained for 30 days as \`test-reports-${{ matrix.profile }}\`
145150
EOF
146151
else
147152
echo "⚠️ Test report file not found!" >> $GITHUB_STEP_SUMMARY

‎.github/workflows/unit-test-e2e-testcases.yml‎

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
name: E2E Testcases Unit Tests
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "e2e-tests/testcases/**"
7+
- "src/semantic-router/pkg/classification/**"
8+
- "src/semantic-router/pkg/config/**"
9+
- "candle-binding/**"
10+
- ".github/workflows/unit-test-e2e-testcases.yml"
11+
push:
12+
branches:
13+
- main
14+
workflow_dispatch:
15+
16+
env:
17+
GO_VERSION: '1.24'
18+
RUST_VERSION: '1.90.0'
19+
20+
jobs:
21+
test-keyword-routing:
22+
name: Keyword Routing Tests
23+
runs-on: ubuntu-latest
24+
25+
steps:
26+
- name: Checkout code
27+
uses: actions/checkout@v4
28+
29+
- name: Set up Go
30+
uses: actions/setup-go@v5
31+
with:
32+
go-version: ${{ env.GO_VERSION }}
33+
cache: true
34+
cache-dependency-path: e2e-tests/testcases/go.sum
35+
36+
- name: Set up Rust
37+
uses: actions-rust-lang/setup-rust-toolchain@v1
38+
with:
39+
toolchain: ${{ env.RUST_VERSION }}
40+
41+
- name: Cache Rust dependencies
42+
uses: actions/cache@v4
43+
with:
44+
path: |
45+
~/.cargo/bin/
46+
~/.cargo/registry/index/
47+
~/.cargo/registry/cache/
48+
~/.cargo/git/db/
49+
candle-binding/target/
50+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
51+
restore-keys: |
52+
${{ runner.os }}-cargo-
53+
54+
- name: Build Rust Candle Bindings
55+
run: |
56+
cd candle-binding
57+
cargo build --release --no-default-features
58+
ls -la target/release/
59+
60+
- name: Verify Rust library
61+
run: |
62+
if [ -f "candle-binding/target/release/libcandle_semantic_router.so" ]; then
63+
echo "✅ Rust library built successfully"
64+
ls -lh candle-binding/target/release/libcandle_semantic_router.so
65+
else
66+
echo "❌ Rust library not found"
67+
exit 1
68+
fi
69+
70+
- name: Run Keyword Routing Tests
71+
env:
72+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
73+
run: |
74+
cd e2e-tests/testcases
75+
echo "Running keyword routing tests..."
76+
go test -v -coverprofile=coverage-keyword.out -covermode=atomic -coverpkg=github.com/vllm-project/semantic-router/src/semantic-router/pkg/classification
77+
78+
- name: Generate coverage report
79+
if: always()
80+
run: |
81+
cd e2e-tests/testcases
82+
go tool cover -func=coverage-keyword.out > coverage-summary.txt
83+
echo "=== Full Coverage Summary ==="
84+
cat coverage-summary.txt
85+
86+
echo ""
87+
echo "=== Keyword Classifier Coverage ==="
88+
grep "keyword_classifier.go" coverage-summary.txt || echo "No keyword_classifier.go coverage found"
89+
90+
# Extract coverage for keyword_classifier.go only
91+
# Filter lines containing keyword_classifier.go, extract percentage, calculate average
92+
KEYWORD_COVERAGE=$(grep "keyword_classifier.go" coverage-summary.txt | awk '{gsub(/%/, "", $NF); sum+=$NF; count++} END {if(count>0) printf "%.1f", sum/count; else print "0.0"}')
93+
echo "Keyword Classifier Average Coverage: ${KEYWORD_COVERAGE}%"
94+
echo "COVERAGE=${KEYWORD_COVERAGE}%" >> $GITHUB_ENV
95+
96+
- name: Check coverage threshold
97+
if: always()
98+
run: |
99+
cd e2e-tests/testcases
100+
COVERAGE_PERCENT=$(echo $COVERAGE | sed 's/%//')
101+
THRESHOLD=80
102+
103+
if (( $(echo "$COVERAGE_PERCENT < $THRESHOLD" | bc -l) )); then
104+
echo "❌ Coverage $COVERAGE is below threshold ${THRESHOLD}%"
105+
exit 1
106+
else
107+
echo "✅ Coverage $COVERAGE meets threshold ${THRESHOLD}%"
108+
fi
109+
110+
- name: Upload coverage to Codecov
111+
uses: codecov/codecov-action@v4
112+
if: always()
113+
with:
114+
files: ./e2e-tests/testcases/coverage-keyword.out
115+
flags: e2e-testcases-keyword
116+
name: keyword-routing-coverage
117+
fail_ci_if_error: false
118+
119+
- name: Test Summary
120+
if: always()
121+
run: |
122+
echo "### Keyword Routing Test Results :test_tube:" >> $GITHUB_STEP_SUMMARY
123+
echo "" >> $GITHUB_STEP_SUMMARY
124+
echo "**Coverage:** $COVERAGE" >> $GITHUB_STEP_SUMMARY
125+
echo "" >> $GITHUB_STEP_SUMMARY
126+
echo "#### Test Categories" >> $GITHUB_STEP_SUMMARY
127+
echo "- ✅ OR operator tests" >> $GITHUB_STEP_SUMMARY
128+
echo "- ✅ AND operator tests" >> $GITHUB_STEP_SUMMARY
129+
echo "- ✅ NOR operator tests" >> $GITHUB_STEP_SUMMARY
130+
echo "- ✅ Case sensitivity tests" >> $GITHUB_STEP_SUMMARY
131+
echo "- ✅ Word boundary tests" >> $GITHUB_STEP_SUMMARY
132+
echo "- ✅ Regex special character tests" >> $GITHUB_STEP_SUMMARY
133+
echo "- ✅ Edge case tests" >> $GITHUB_STEP_SUMMARY
134+
echo "- ✅ Multiple rule matching" >> $GITHUB_STEP_SUMMARY
135+
echo "- ✅ Confidence score validation" >> $GITHUB_STEP_SUMMARY
136+
echo "- ✅ JSON test data loading" >> $GITHUB_STEP_SUMMARY
137+
echo "- ✅ Error handling" >> $GITHUB_STEP_SUMMARY
138+
139+
test-embedding-routing:
140+
name: Embedding Routing Tests
141+
runs-on: ubuntu-latest
142+
# Only run if embedding tests exist (for future PRs)
143+
if: |
144+
contains(github.event.pull_request.changed_files, 'e2e-tests/testcases/embedding_routing_test.go') ||
145+
github.event_name == 'workflow_dispatch'
146+
147+
steps:
148+
- name: Checkout code
149+
uses: actions/checkout@v4
150+
151+
- name: Set up Go
152+
uses: actions/setup-go@v5
153+
with:
154+
go-version: ${{ env.GO_VERSION }}
155+
cache: true
156+
157+
- name: Set up Rust
158+
uses: actions-rust-lang/setup-rust-toolchain@v1
159+
with:
160+
toolchain: ${{ env.RUST_VERSION }}
161+
162+
- name: Build Rust Candle Bindings
163+
run: |
164+
cd candle-binding
165+
cargo build --release --no-default-features
166+
167+
- name: Run Embedding Routing Tests
168+
env:
169+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
170+
run: |
171+
cd e2e-tests/testcases
172+
if [ -f "embedding_routing_test.go" ] && ! [[ "$(basename embedding_routing_test.go)" =~ \.skip$ ]]; then
173+
echo "Running embedding routing tests..."
174+
go test -v -run "Embedding Routing" -coverprofile=coverage-embedding.out -covermode=atomic
175+
else
176+
echo "⏭️ Embedding routing tests not ready yet (skipped)"
177+
fi
178+
179+
test-hybrid-routing:
180+
name: Hybrid Routing Tests
181+
runs-on: ubuntu-latest
182+
# Only run if hybrid tests exist (for future PRs)
183+
if: |
184+
contains(github.event.pull_request.changed_files, 'e2e-tests/testcases/hybrid_routing_test.go') ||
185+
github.event_name == 'workflow_dispatch'
186+
187+
steps:
188+
- name: Checkout code
189+
uses: actions/checkout@v4
190+
191+
- name: Set up Go
192+
uses: actions/setup-go@v5
193+
with:
194+
go-version: ${{ env.GO_VERSION }}
195+
cache: true
196+
197+
- name: Set up Rust
198+
uses: actions-rust-lang/setup-rust-toolchain@v1
199+
with:
200+
toolchain: ${{ env.RUST_VERSION }}
201+
202+
- name: Build Rust Candle Bindings
203+
run: |
204+
cd candle-binding
205+
cargo build --release --no-default-features
206+
207+
- name: Run Hybrid Routing Tests
208+
env:
209+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
210+
run: |
211+
cd e2e-tests/testcases
212+
if [ -f "hybrid_routing_test.go" ] && ! [[ "$(basename hybrid_routing_test.go)" =~ \.skip$ ]]; then
213+
echo "Running hybrid routing tests..."
214+
go test -v -run "Hybrid Routing" -coverprofile=coverage-hybrid.out -covermode=atomic
215+
else
216+
echo "⏭️ Hybrid routing tests not ready yet (skipped)"
217+
fi
218+
219+
race-detection:
220+
name: Race Condition Detection
221+
runs-on: ubuntu-latest
222+
223+
steps:
224+
- name: Checkout code
225+
uses: actions/checkout@v4
226+
227+
- name: Set up Go
228+
uses: actions/setup-go@v5
229+
with:
230+
go-version: ${{ env.GO_VERSION }}
231+
cache: true
232+
233+
- name: Set up Rust
234+
uses: actions-rust-lang/setup-rust-toolchain@v1
235+
with:
236+
toolchain: ${{ env.RUST_VERSION }}
237+
238+
- name: Build Rust Candle Bindings
239+
run: |
240+
cd candle-binding
241+
cargo build --release --no-default-features
242+
243+
- name: Run tests with race detector
244+
env:
245+
LD_LIBRARY_PATH: ${{ github.workspace }}/candle-binding/target/release
246+
run: |
247+
cd e2e-tests/testcases
248+
echo "Running tests with race detector..."
249+
go test -race -v || {
250+
echo "❌ Race conditions detected!"
251+
exit 1
252+
}
253+
echo "✅ No race conditions detected"
254+
255+
lint:
256+
name: Lint Go Code
257+
runs-on: ubuntu-latest
258+
259+
steps:
260+
- name: Checkout code
261+
uses: actions/checkout@v4
262+
263+
- name: Set up Go
264+
uses: actions/setup-go@v5
265+
with:
266+
go-version: ${{ env.GO_VERSION }}
267+
cache: true
268+
269+
- name: Run golangci-lint
270+
uses: golangci/golangci-lint-action@v6
271+
with:
272+
version: latest
273+
working-directory: e2e-tests/testcases
274+
args: --timeout=5m
275+
276+
summary:
277+
name: Test Summary
278+
if: always()
279+
runs-on: ubuntu-latest
280+
needs: [test-keyword-routing, race-detection, lint]
281+
282+
steps:
283+
- name: Check test results
284+
run: |
285+
echo "=== E2E Testcases Summary ==="
286+
echo "Keyword Routing Tests: ${{ needs.test-keyword-routing.result }}"
287+
echo "Race Detection: ${{ needs.race-detection.result }}"
288+
echo "Lint: ${{ needs.lint.result }}"
289+
290+
# Count failures
291+
FAILURES=0
292+
if [[ "${{ needs.test-keyword-routing.result }}" == "failure" ]]; then
293+
echo "❌ Keyword routing tests failed"
294+
FAILURES=$((FAILURES + 1))
295+
fi
296+
if [[ "${{ needs.race-detection.result }}" == "failure" ]]; then
297+
echo "❌ Race detection failed"
298+
FAILURES=$((FAILURES + 1))
299+
fi
300+
if [[ "${{ needs.lint.result }}" == "failure" ]]; then
301+
echo "❌ Lint failed"
302+
FAILURES=$((FAILURES + 1))
303+
fi
304+
305+
echo ""
306+
echo "=== Test Coverage (Issue #667) ==="
307+
echo "✅ OR operator - any keyword matches"
308+
echo "✅ AND operator - all keywords must match"
309+
echo "✅ NOR operator - no keywords match"
310+
echo "✅ Case-sensitive vs case-insensitive matching"
311+
echo "✅ Regex pattern matching"
312+
echo "✅ Word boundary detection"
313+
echo "✅ Priority over embedding and intent-based routing"
314+
315+
if [ $FAILURES -gt 0 ]; then
316+
echo ""
317+
echo "❌ $FAILURES test(s) failed. Check the logs for details."
318+
exit 1
319+
else
320+
echo ""
321+
echo "✅ All E2E testcases passed!"
322+
fi

0 commit comments

Comments
 (0)