From 047fa67857e3cfc5067841e7628ee1d585dce747 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Fri, 4 Sep 2026 15:07:02 +0200 Subject: [PATCH 1/6] LCORE-1673: e2e feature file for conversation compaction (no step implementation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author the behave scenarios for conversation compaction from the design document alone, ahead of the step definitions (LCORE-2230), so the test shape follows the intended behaviour rather than the implementation. The scenarios observe compaction strictly from outside the deployed stack — e2e steps never touch src/ (docs/testing/e2e_testing.md, "Choosing the Test Layer") — which leaves exactly the surfaces the design exposes over HTTP: - context_status on /v1/query responses: "full" while compaction never triggers, "summarized" once a three-turn conversation crosses the configured threshold (R7, driven by R1/R9 configuration). - The assistant still recalls a fact stated before the summary. - The Conversations API keeps serving every user query after compaction (R6). - /v1/streaming_query emits a compaction event before the first token and carries context_status in its end event (R12, R7). - enabled: false is a full off-switch: context_status stays "full" past the threshold. Buffer semantics, additive summarization, the summarization model and prompt, tiktoken estimation and per-conversation blocking are internal and belong to the integration layer (LCORE-1574). Fixtures: lightspeed-stack-compaction.yaml and lightspeed-stack-compaction-disabled.yaml per mode, derived from the mode's base configuration. They register a deliberately small context window for the CI model (openai/gpt-4o-mini: 2000 tokens) and a 10% threshold above a 100-token floor, so three turns cross the trigger; the real provider window is untouched, this only drives the local estimate. The window key is model-specific, so the scenarios that need it carry @openai-only, the same gating the unified-mode boot scenarios use on the providers matrix. tests/e2e/test_list.txt gains the feature; behave --dry-run parses it with the nine new step patterns reported as undefined and every other step matched. --- .../lightspeed-stack-compaction-disabled.yaml | 62 +++++++ .../lightspeed-stack-compaction.yaml | 61 +++++++ .../lightspeed-stack-compaction-disabled.yaml | 60 +++++++ .../lightspeed-stack-compaction.yaml | 59 +++++++ .../features/conversation-compaction.feature | 165 ++++++++++++++++++ tests/e2e/test_list.txt | 1 + 6 files changed, 408 insertions(+) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml create mode 100644 tests/e2e/features/conversation-compaction.feature diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml new file mode 100644 index 000000000..a38ef5e9d --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml @@ -0,0 +1,62 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +ogx: + # Library mode - embeds OGX as library + use_as_library_client: true + # Unified mode: run.yaml (materialized per provider by CI/the harness) + # is consumed as the synthesis profile instead of the legacy two-file path. + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Compaction e2e (LCORE-1673): a deliberately small window for the CI + # model so a three-turn conversation crosses the trigger threshold. + # The real model window is far larger; this only drives the local + # estimate, never the provider. + context_windows: + openai/gpt-4o-mini: 2000 +rag: + byok: + stores: + - rag_id: e2e-test-docs + backend: faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + retrieval: + tool: + sources: + - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + +# Same small window and threshold as lightspeed-stack-compaction.yaml, but +# compaction switched off: context_status must stay "full" past the +# threshold (enabled is a full off-switch). +compaction: + enabled: false + threshold_ratio: 0.1 + token_floor: 100 + buffer_turns: 1 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml new file mode 100644 index 000000000..347058e1b --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml @@ -0,0 +1,61 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +ogx: + # Library mode - embeds OGX as library + use_as_library_client: true + # Unified mode: run.yaml (materialized per provider by CI/the harness) + # is consumed as the synthesis profile instead of the legacy two-file path. + config: + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Compaction e2e (LCORE-1673): a deliberately small window for the CI + # model so a three-turn conversation crosses the trigger threshold. + # The real model window is far larger; this only drives the local + # estimate, never the provider. + context_windows: + openai/gpt-4o-mini: 2000 +rag: + byok: + stores: + - rag_id: e2e-test-docs + backend: faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + retrieval: + tool: + sources: + - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + +# Compaction on with a low threshold: 10% of the 2000-token window, +# above a 100-token floor, keeping one recent turn verbatim. +compaction: + enabled: true + threshold_ratio: 0.1 + token_floor: 100 + buffer_turns: 1 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml new file mode 100644 index 000000000..86b08fb7e --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml @@ -0,0 +1,60 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +ogx: + # Server mode - connects to separate OGX service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Compaction e2e (LCORE-1673): a deliberately small window for the CI + # model so a three-turn conversation crosses the trigger threshold. + # The real model window is far larger; this only drives the local + # estimate, never the provider. + context_windows: + openai/gpt-4o-mini: 2000 +rag: + byok: + stores: + - rag_id: e2e-test-docs + backend: faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + retrieval: + tool: + sources: + - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + +# Same small window and threshold as lightspeed-stack-compaction.yaml, but +# compaction switched off: context_status must stay "full" past the +# threshold (enabled is a full off-switch). +compaction: + enabled: false + threshold_ratio: 0.1 + token_floor: 100 + buffer_turns: 1 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml new file mode 100644 index 000000000..c2a156586 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml @@ -0,0 +1,59 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +ogx: + # Server mode - connects to separate OGX service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Compaction e2e (LCORE-1673): a deliberately small window for the CI + # model so a three-turn conversation crosses the trigger threshold. + # The real model window is far larger; this only drives the local + # estimate, never the provider. + context_windows: + openai/gpt-4o-mini: 2000 +rag: + byok: + stores: + - rag_id: e2e-test-docs + backend: faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + retrieval: + tool: + sources: + - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + +# Compaction on with a low threshold: 10% of the 2000-token window, +# above a 100-token floor, keeping one recent turn verbatim. +compaction: + enabled: true + threshold_ratio: 0.1 + token_floor: 100 + buffer_turns: 1 diff --git a/tests/e2e/features/conversation-compaction.feature b/tests/e2e/features/conversation-compaction.feature new file mode 100644 index 000000000..cd00ca03a --- /dev/null +++ b/tests/e2e/features/conversation-compaction.feature @@ -0,0 +1,165 @@ +@cfg_compaction +Feature: Conversation compaction + + When a conversation's estimated input approaches the model's context + window, older turns are summarized before the request reaches the LLM + (docs/design/conversation-compaction/conversation-compaction.md). These + scenarios observe compaction from outside only: the context_status field + on responses (R7), the compaction event on the native stream (R12), the + full history the Conversations API keeps serving (R6), and the assistant's + recall of what was said before the summary. The trigger is driven by the + admin configuration (R1, R9): the compaction fixtures register a small + context window for the openai model and a low threshold ratio, so a + three-turn conversation crosses it. + + Background: + Given The service is started locally + And The system is in default state + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + + + Scenario: context_status reports full while compaction never triggers + Given The service uses the lightspeed-stack.yaml configuration + And The service is restarted + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "full" + + + @openai-only + Scenario: context_status reports summarized once the conversation crosses the threshold + Given The service uses the lightspeed-stack-compaction.yaml configuration + And The service is restarted + When I use "query" to ask question + """ + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I store conversation details + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "summarized" + + + @openai-only + Scenario: the assistant still recalls what was said before the summary + Given The service uses the lightspeed-stack-compaction.yaml configuration + And The service is restarted + When I use "query" to ask question + """ + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I store conversation details + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "summarized" + When I use "query" to ask question with same conversation_id + """ + {"query": "What is the name of my cluster? Reply with the name only.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "summarized" + And The response contains following fragments + | Fragments in LLM response | + | aurora-prod-7 | + + + @openai-only + Scenario: the full conversation history stays available after compaction + Given The service uses the lightspeed-stack-compaction.yaml configuration + And The service is restarted + When I use "query" to ask question + """ + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I store conversation details + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "summarized" + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 200 + And The conversation history includes the following user queries + | User query | + | My OpenShift cluster is named aurora-prod-7. Remember that name. | + | Explain what a pod is in about five sentences. | + | Explain what a deployment is in about five sentences. | + + + @openai-only + Scenario: the native stream announces compaction and reports context_status + Given The service uses the lightspeed-stack-compaction.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question + """ + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And I store conversation details + When I use "streaming_query" to ask question with same conversation_id + """ + {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + When I use "streaming_query" to ask question with same conversation_id + """ + {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The streamed response contains a compaction event before the first token + And The streamed response end event has context_status "summarized" + + + @openai-only + Scenario: compaction stays off when disabled, even past the threshold + Given The service uses the lightspeed-stack-compaction-disabled.yaml configuration + And The service is restarted + When I use "query" to ask question + """ + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I store conversation details + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + When I use "query" to ask question with same conversation_id + """ + {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "full" diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 79f34fcb5..c37e04e98 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -23,6 +23,7 @@ features/rlsapi_v1.feature features/streaming_query.feature features/vector_stores.feature features/conversation_cache_v2.feature +features/conversation-compaction.feature features/feedback.feature features/http_401_unauthorized.feature features/rbac.feature From a547bf2791b32643aee6dbee48c55998b4c6180f Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Fri, 11 Sep 2026 15:01:35 +0200 Subject: [PATCH 2/6] LCORE-1673: make the compaction crossing deterministic and cut the feature to three scenarios Review rework of the conversation-compaction feature file. Scenarios. The six scenarios collapse into three. The base-config "full" scenario duplicated the disabled one and is gone. The "summarized", "recall" and "history" scenarios exercised the same three turns and are now one scenario; the streaming and disabled scenarios stay separate because they observe different surfaces (the SSE stream, the off switch). Deterministic trigger. The old scenarios relied on three ordinary queries adding up to the threshold, so the crossing turn depended on response length. Now the first two turns ask the model to reply with "OK" only and each plants one fact, and the third query is a fixed 248-token paragraph that crosses the 200-token threshold on its own (2000-token window, ratio 0.1, floor 100). With buffer_turns 1 nothing can be summarized before the third turn, so context_status is asserted "full" on turns one and two and "summarized" on turn three. The third query also asks for both facts: turn one is in the summary, turn two is the verbatim buffer, so one answer covers summary recall and buffer recall. Fixtures. The pii-redaction shield is dropped: it cost tokens on every turn and its \d+ rule would have rewritten the cluster name the recall assertion looks for. context_windows now lists every provider/model pair the e2e workflows run (openai, azure, google-vertex, watsonx, aws-bedrock) instead of only openai, and the @openai-only tags go away with it. The vLLM runs take their model id from an env var and context_windows keys are not env-substituted, so they are not listed. Skip. The feature is tagged @skip at feature level until LCORE-2230 lands the nine step definitions; Konflux runs the whole test list and would fail on undefined steps. The description drops the R-references. Rebased onto main after the OGX rename: the server-mode fixtures now read the OGX host from E2E_OGX_HOSTNAME like the base fixture does. --- .../lightspeed-stack-compaction-disabled.yaml | 23 ++-- .../lightspeed-stack-compaction.yaml | 23 ++-- .../lightspeed-stack-compaction-disabled.yaml | 25 ++-- .../lightspeed-stack-compaction.yaml | 25 ++-- .../features/conversation-compaction.feature | 121 +++++------------- 5 files changed, 75 insertions(+), 142 deletions(-) diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml index a38ef5e9d..227792a2f 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction-disabled.yaml @@ -23,12 +23,19 @@ authentication: inference: default_provider: openai default_model: gpt-4o-mini - # Compaction e2e (LCORE-1673): a deliberately small window for the CI - # model so a three-turn conversation crosses the trigger threshold. - # The real model window is far larger; this only drives the local - # estimate, never the provider. + # Compaction e2e (LCORE-1673): a deliberately small window for every + # model the e2e workflows run against, so the third query of the + # compaction scenarios crosses the trigger threshold. The real windows + # are far larger; this only drives the local estimate, never the provider. + # The vLLM-backed runs (rhaiis, rhelai) take the model id from an env + # var, and context_windows keys are not env-substituted, so they are + # not listed and skip the token-based trigger. context_windows: openai/gpt-4o-mini: 2000 + azure/gpt-4o-mini: 2000 + google-vertex/publishers/google/models/gemini-2.5-flash: 2000 + watsonx/meta-llama/llama-3-3-70b-instruct: 2000 + aws-bedrock/deepseek.v3-v1:0: 2000 rag: byok: stores: @@ -44,14 +51,6 @@ rag: sources: - e2e-test-docs -shields: - - name: pii-redaction - provider_id: redaction - config: - rules: - - pattern: '\d+' - replacement: '[NUM]' - # Same small window and threshold as lightspeed-stack-compaction.yaml, but # compaction switched off: context_status must stay "full" past the # threshold (enabled is a full off-switch). diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml index 347058e1b..94675e16c 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-compaction.yaml @@ -23,12 +23,19 @@ authentication: inference: default_provider: openai default_model: gpt-4o-mini - # Compaction e2e (LCORE-1673): a deliberately small window for the CI - # model so a three-turn conversation crosses the trigger threshold. - # The real model window is far larger; this only drives the local - # estimate, never the provider. + # Compaction e2e (LCORE-1673): a deliberately small window for every + # model the e2e workflows run against, so the third query of the + # compaction scenarios crosses the trigger threshold. The real windows + # are far larger; this only drives the local estimate, never the provider. + # The vLLM-backed runs (rhaiis, rhelai) take the model id from an env + # var, and context_windows keys are not env-substituted, so they are + # not listed and skip the token-based trigger. context_windows: openai/gpt-4o-mini: 2000 + azure/gpt-4o-mini: 2000 + google-vertex/publishers/google/models/gemini-2.5-flash: 2000 + watsonx/meta-llama/llama-3-3-70b-instruct: 2000 + aws-bedrock/deepseek.v3-v1:0: 2000 rag: byok: stores: @@ -44,14 +51,6 @@ rag: sources: - e2e-test-docs -shields: - - name: pii-redaction - provider_id: redaction - config: - rules: - - pattern: '\d+' - replacement: '[NUM]' - # Compaction on with a low threshold: 10% of the 2000-token window, # above a 100-token floor, keeping one recent turn verbatim. compaction: diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml index 86b08fb7e..ff1e3510c 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction-disabled.yaml @@ -9,7 +9,7 @@ service: ogx: # Server mode - connects to separate OGX service use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + url: http://${env.E2E_OGX_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true @@ -21,12 +21,19 @@ authentication: inference: default_provider: openai default_model: gpt-4o-mini - # Compaction e2e (LCORE-1673): a deliberately small window for the CI - # model so a three-turn conversation crosses the trigger threshold. - # The real model window is far larger; this only drives the local - # estimate, never the provider. + # Compaction e2e (LCORE-1673): a deliberately small window for every + # model the e2e workflows run against, so the third query of the + # compaction scenarios crosses the trigger threshold. The real windows + # are far larger; this only drives the local estimate, never the provider. + # The vLLM-backed runs (rhaiis, rhelai) take the model id from an env + # var, and context_windows keys are not env-substituted, so they are + # not listed and skip the token-based trigger. context_windows: openai/gpt-4o-mini: 2000 + azure/gpt-4o-mini: 2000 + google-vertex/publishers/google/models/gemini-2.5-flash: 2000 + watsonx/meta-llama/llama-3-3-70b-instruct: 2000 + aws-bedrock/deepseek.v3-v1:0: 2000 rag: byok: stores: @@ -42,14 +49,6 @@ rag: sources: - e2e-test-docs -shields: - - name: pii-redaction - provider_id: redaction - config: - rules: - - pattern: '\d+' - replacement: '[NUM]' - # Same small window and threshold as lightspeed-stack-compaction.yaml, but # compaction switched off: context_status must stay "full" past the # threshold (enabled is a full off-switch). diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml index c2a156586..82dd0c32b 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-compaction.yaml @@ -9,7 +9,7 @@ service: ogx: # Server mode - connects to separate OGX service use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + url: http://${env.E2E_OGX_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true @@ -21,12 +21,19 @@ authentication: inference: default_provider: openai default_model: gpt-4o-mini - # Compaction e2e (LCORE-1673): a deliberately small window for the CI - # model so a three-turn conversation crosses the trigger threshold. - # The real model window is far larger; this only drives the local - # estimate, never the provider. + # Compaction e2e (LCORE-1673): a deliberately small window for every + # model the e2e workflows run against, so the third query of the + # compaction scenarios crosses the trigger threshold. The real windows + # are far larger; this only drives the local estimate, never the provider. + # The vLLM-backed runs (rhaiis, rhelai) take the model id from an env + # var, and context_windows keys are not env-substituted, so they are + # not listed and skip the token-based trigger. context_windows: openai/gpt-4o-mini: 2000 + azure/gpt-4o-mini: 2000 + google-vertex/publishers/google/models/gemini-2.5-flash: 2000 + watsonx/meta-llama/llama-3-3-70b-instruct: 2000 + aws-bedrock/deepseek.v3-v1:0: 2000 rag: byok: stores: @@ -42,14 +49,6 @@ rag: sources: - e2e-test-docs -shields: - - name: pii-redaction - provider_id: redaction - config: - rules: - - pattern: '\d+' - replacement: '[NUM]' - # Compaction on with a low threshold: 10% of the 2000-token window, # above a 100-token floor, keeping one recent turn verbatim. compaction: diff --git a/tests/e2e/features/conversation-compaction.feature b/tests/e2e/features/conversation-compaction.feature index cd00ca03a..89d794788 100644 --- a/tests/e2e/features/conversation-compaction.feature +++ b/tests/e2e/features/conversation-compaction.feature @@ -1,16 +1,15 @@ -@cfg_compaction +# @skip until LCORE-2230 lands the step definitions; Konflux runs the whole +# test list and would fail on the undefined steps. @cfg_compaction is not in +# any GitHub CI shard yet, LCORE-2230 adds it. +@cfg_compaction @skip Feature: Conversation compaction - When a conversation's estimated input approaches the model's context - window, older turns are summarized before the request reaches the LLM - (docs/design/conversation-compaction/conversation-compaction.md). These - scenarios observe compaction from outside only: the context_status field - on responses (R7), the compaction event on the native stream (R12), the - full history the Conversations API keeps serving (R6), and the assistant's - recall of what was said before the summary. The trigger is driven by the - admin configuration (R1, R9): the compaction fixtures register a small - context window for the openai model and a low threshold ratio, so a - three-turn conversation crosses it. + Once the estimated input crosses the configured share of the model's + context window, older turns are summarized before the request reaches + the model. The compaction fixtures register a 2000-token window with a + 10% threshold and keep one recent turn verbatim, so a long third query + is what crosses it: turn one ends up in the summary, turn two stays in + the verbatim buffer, and the third query asks for a fact from each. Background: Given The service is started locally @@ -19,122 +18,61 @@ Feature: Conversation compaction And the Lightspeed stack configuration directory is "tests/e2e/configuration" - Scenario: context_status reports full while compaction never triggers - Given The service uses the lightspeed-stack.yaml configuration - And The service is restarted - When I use "query" to ask question - """ - {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The response context_status is "full" - - - @openai-only - Scenario: context_status reports summarized once the conversation crosses the threshold - Given The service uses the lightspeed-stack-compaction.yaml configuration - And The service is restarted - When I use "query" to ask question - """ - {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And I store conversation details - When I use "query" to ask question with same conversation_id - """ - {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - When I use "query" to ask question with same conversation_id - """ - {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The response context_status is "summarized" - - - @openai-only - Scenario: the assistant still recalls what was said before the summary + Scenario: the third query crosses the threshold, older turns are summarized, recall and history survive Given The service uses the lightspeed-stack-compaction.yaml configuration And The service is restarted When I use "query" to ask question """ - {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 + And The response context_status is "full" And I store conversation details When I use "query" to ask question with same conversation_id """ - {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "My application namespace is called blue-lagoon. Remember that name too and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 + And The response context_status is "full" When I use "query" to ask question with same conversation_id """ - {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The response context_status is "summarized" - When I use "query" to ask question with same conversation_id - """ - {"query": "What is the name of my cluster? Reply with the name only.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Some background on my environment first, no need to comment on it. The cluster runs on bare metal in two racks with three control plane nodes and nine worker nodes, all on the same subnet behind a pair of hardware load balancers. Storage is provided by an external Ceph cluster exposed through the CSI driver, with three storage classes for block, file and object access. Ingress is handled by the default router with two replicas pinned to the infra nodes, and TLS certificates are issued by an internal certificate authority and rotated every ninety days. Monitoring uses the built-in Prometheus stack with a remote write to a central Thanos instance, and alerts are routed to an on-call rotation through a webhook receiver. The image registry is the internal one, backed by an object storage bucket, and images are mirrored from an upstream registry once a day by a scheduled job. Upgrades follow the stable channel, one minor version at a time, and are rehearsed on a staging cluster of the same shape a week before production. Backups of etcd are taken hourly and copied off-site nightly. Now the question: what is the name of my cluster and what is the name of my application namespace? Reply with the two names only, separated by a comma.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The response context_status is "summarized" And The response contains following fragments | Fragments in LLM response | | aurora-prod-7 | - - - @openai-only - Scenario: the full conversation history stays available after compaction - Given The service uses the lightspeed-stack-compaction.yaml configuration - And The service is restarted - When I use "query" to ask question - """ - {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And I store conversation details - When I use "query" to ask question with same conversation_id - """ - {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - When I use "query" to ask question with same conversation_id - """ - {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The response context_status is "summarized" + | blue-lagoon | When I use REST API conversation endpoint with conversation_id from above using HTTP GET method Then The status code of the response is 200 And The conversation history includes the following user queries - | User query | - | My OpenShift cluster is named aurora-prod-7. Remember that name. | - | Explain what a pod is in about five sentences. | - | Explain what a deployment is in about five sentences. | + | User query | + | My OpenShift cluster is named aurora-prod-7. Remember that name and reply with OK only. | + | My application namespace is called blue-lagoon. Remember that name too and reply with OK only. | - @openai-only - Scenario: the native stream announces compaction and reports context_status + Scenario: the native stream announces compaction on the query that crosses the threshold Given The service uses the lightspeed-stack-compaction.yaml configuration And The service is restarted When I use "streaming_query" to ask question """ - {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And I wait for the response to be completed + And The streamed response end event has context_status "full" And I store conversation details When I use "streaming_query" to ask question with same conversation_id """ - {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "My application namespace is called blue-lagoon. Remember that name too and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And I wait for the response to be completed + And The streamed response end event has context_status "full" When I use "streaming_query" to ask question with same conversation_id """ - {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Some background on my environment first, no need to comment on it. The cluster runs on bare metal in two racks with three control plane nodes and nine worker nodes, all on the same subnet behind a pair of hardware load balancers. Storage is provided by an external Ceph cluster exposed through the CSI driver, with three storage classes for block, file and object access. Ingress is handled by the default router with two replicas pinned to the infra nodes, and TLS certificates are issued by an internal certificate authority and rotated every ninety days. Monitoring uses the built-in Prometheus stack with a remote write to a central Thanos instance, and alerts are routed to an on-call rotation through a webhook receiver. The image registry is the internal one, backed by an object storage bucket, and images are mirrored from an upstream registry once a day by a scheduled job. Upgrades follow the stable channel, one minor version at a time, and are rehearsed on a staging cluster of the same shape a week before production. Backups of etcd are taken hourly and copied off-site nightly. Now the question: what is the name of my cluster and what is the name of my application namespace? Reply with the two names only, separated by a comma.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And I wait for the response to be completed @@ -142,24 +80,23 @@ Feature: Conversation compaction And The streamed response end event has context_status "summarized" - @openai-only Scenario: compaction stays off when disabled, even past the threshold Given The service uses the lightspeed-stack-compaction-disabled.yaml configuration And The service is restarted When I use "query" to ask question """ - {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And I store conversation details When I use "query" to ask question with same conversation_id """ - {"query": "Explain what a pod is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "My application namespace is called blue-lagoon. Remember that name too and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 When I use "query" to ask question with same conversation_id """ - {"query": "Explain what a deployment is in about five sentences.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Some background on my environment first, no need to comment on it. The cluster runs on bare metal in two racks with three control plane nodes and nine worker nodes, all on the same subnet behind a pair of hardware load balancers. Storage is provided by an external Ceph cluster exposed through the CSI driver, with three storage classes for block, file and object access. Ingress is handled by the default router with two replicas pinned to the infra nodes, and TLS certificates are issued by an internal certificate authority and rotated every ninety days. Monitoring uses the built-in Prometheus stack with a remote write to a central Thanos instance, and alerts are routed to an on-call rotation through a webhook receiver. The image registry is the internal one, backed by an object storage bucket, and images are mirrored from an upstream registry once a day by a scheduled job. Upgrades follow the stable channel, one minor version at a time, and are rehearsed on a staging cluster of the same shape a week before production. Backups of etcd are taken hourly and copied off-site nightly. Now the question: what is the name of my cluster and what is the name of my application namespace? Reply with the two names only, separated by a comma.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The response context_status is "full" From b8ca222ed42aa5e1d69e4cea56a59cc1fb06ae23 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Mon, 14 Sep 2026 16:20:02 +0200 Subject: [PATCH 3/6] LCORE-1673: re-trigger CI Empty commit that changes no files. It re-runs the PR checks after the build-pr image build failed on a network error: uv could not download jiter==0.16.0 after 3 retries. PR contributors cannot re-run fork-originated workflow runs directly. From b5c39aa3f0fc5088b508c9f4ac81526abbf84d0b Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Thu, 17 Sep 2026 17:57:22 +0200 Subject: [PATCH 4/6] LCORE-1673: cover a second compaction in the query scenario Review asked for the scenario to check that a second compaction keeps the first summary instead of overwriting it. The query scenario gains two queries after the threshold crossing: - Query 4 states a database name (green-harbor). Only one turn follows the first summary marker and it fits the one-turn buffer, so nothing is summarized; context_status stays "summarized". - Query 5 asks for the datacenter and database names. Turns 3 and 4 now follow the marker, so turn 3 is summarized into a second summary and turn 4 stays verbatim. The response must contain north-quarry and green-harbor. Query 1 now also states the datacenter name (north-quarry). The first summary is the only place that name exists: query 3 asks for the cluster and namespace names only, so its answer never repeats it. A check on aurora-prod-7 would not prove anything here, because query 3's answer repeats it and query 5 summarizes that answer into the second summary. The history table follows the new query 1 text. Checked by running the scenario's exact queries through the real apply_compaction with the conversation store and the summarization call mocked (2000-token window, threshold 0.1, floor 100, one buffer turn): the second summary is created on query 5 and none on query 4, and at query 5 north-quarry is only in the first summary. Buffered turns are not checked beyond query 3: after a compaction they are dropped from later requests, tracked as LCORE-4219. --- .../features/conversation-compaction.feature | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/e2e/features/conversation-compaction.feature b/tests/e2e/features/conversation-compaction.feature index 89d794788..8d553a241 100644 --- a/tests/e2e/features/conversation-compaction.feature +++ b/tests/e2e/features/conversation-compaction.feature @@ -10,6 +10,12 @@ Feature: Conversation compaction 10% threshold and keep one recent turn verbatim, so a long third query is what crosses it: turn one ends up in the summary, turn two stays in the verbatim buffer, and the third query asks for a fact from each. + A fourth query states one more fact without summarizing anything (only + one turn follows the summary), and the fifth summarizes the long third + turn into a second summary. It asks for the datacenter name, which only + the first summary holds (the third query asks for the other two names, + so its answer never repeats it), so the second summary must not replace + the first. Background: Given The service is started locally @@ -18,12 +24,12 @@ Feature: Conversation compaction And the Lightspeed stack configuration directory is "tests/e2e/configuration" - Scenario: the third query crosses the threshold, older turns are summarized, recall and history survive + Scenario: the third query crosses the threshold, a second summary keeps the first, recall and history survive Given The service uses the lightspeed-stack-compaction.yaml configuration And The service is restarted When I use "query" to ask question """ - {"query": "My OpenShift cluster is named aurora-prod-7. Remember that name and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "My OpenShift cluster is named aurora-prod-7 and it runs in the datacenter called north-quarry. Remember both names and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The response context_status is "full" @@ -44,12 +50,28 @@ Feature: Conversation compaction | Fragments in LLM response | | aurora-prod-7 | | blue-lagoon | + When I use "query" to ask question with same conversation_id + """ + {"query": "My database is called green-harbor. Remember that name too and reply with OK only.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "summarized" + When I use "query" to ask question with same conversation_id + """ + {"query": "What is the name of my datacenter and what is the name of my database? Reply with the two names only, separated by a comma.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response context_status is "summarized" + And The response contains following fragments + | Fragments in LLM response | + | north-quarry | + | green-harbor | When I use REST API conversation endpoint with conversation_id from above using HTTP GET method Then The status code of the response is 200 And The conversation history includes the following user queries - | User query | - | My OpenShift cluster is named aurora-prod-7. Remember that name and reply with OK only. | - | My application namespace is called blue-lagoon. Remember that name too and reply with OK only. | + | User query | + | My OpenShift cluster is named aurora-prod-7 and it runs in the datacenter called north-quarry. Remember both names and reply with OK only. | + | My application namespace is called blue-lagoon. Remember that name too and reply with OK only. | Scenario: the native stream announces compaction on the query that crosses the threshold From e1245f7b623c018e3c3bdb927092b81d76a8c5a8 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Fri, 4 Sep 2026 19:37:21 +0200 Subject: [PATCH 5/6] LCORE-2230: behave step definitions for the conversation-compaction feature Implements the step patterns the LCORE-1673 feature file left undefined, observing compaction from outside the deployed stack only: the context_status field on query responses, the compaction event and the end event's context_status on the native stream, and the user queries the Conversations API keeps serving after compaction. Nothing here imports or runs anything under src/; the internals are integration tests (LCORE-1574). The feature loses its @skip tag and @cfg_compaction joins the "other" GitHub shard and the Makefile's default tag expression, so the scenarios run in CI from this change on. One gate step, "the active model has a registered context window", skips a scenario when the active provider/model has no entry under inference.context_windows in the copied lightspeed-stack.yaml. The provider workflows run the whole test list, and the vLLM-backed runs (rhaiis, rhelai) take their model id from an env var that mapping keys cannot pick up, so without the gate those runs would wait for a "summarized" status that can never arrive. --- .github/workflows/e2e_tests.yaml | 2 +- Makefile | 2 +- .../features/conversation-compaction.feature | 8 +- tests/e2e/features/steps/README.md | 4 + .../features/steps/conversation_compaction.py | 157 ++++++++++++++++++ 5 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/features/steps/conversation_compaction.py diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 5e14d19cf..b5c0fe76e 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -36,7 +36,7 @@ jobs: - name: shields tags: "not @skip and @cfg_shields" - name: other - tags: "not @skip and (@cfg_rh_identity or @cfg_negative or @cfg_byok_pdf or @cfg_degraded or @cfg_unified)" + tags: "not @skip and (@cfg_rh_identity or @cfg_negative or @cfg_byok_pdf or @cfg_degraded or @cfg_unified or @cfg_compaction)" # Server-only; listed in shard (not matrix.include) so it expands with # mode=server before any library jobs. include would append after library. - name: tls diff --git a/Makefile b/Makefile index 2fa421ada..44beb3a48 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper) # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g. # E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local -E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) +E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified or @cfg_compaction) test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt" diff --git a/tests/e2e/features/conversation-compaction.feature b/tests/e2e/features/conversation-compaction.feature index 8d553a241..a1fefea9b 100644 --- a/tests/e2e/features/conversation-compaction.feature +++ b/tests/e2e/features/conversation-compaction.feature @@ -1,7 +1,4 @@ -# @skip until LCORE-2230 lands the step definitions; Konflux runs the whole -# test list and would fail on the undefined steps. @cfg_compaction is not in -# any GitHub CI shard yet, LCORE-2230 adds it. -@cfg_compaction @skip +@cfg_compaction Feature: Conversation compaction Once the estimated input crosses the configured share of the model's @@ -26,6 +23,7 @@ Feature: Conversation compaction Scenario: the third query crosses the threshold, a second summary keeps the first, recall and history survive Given The service uses the lightspeed-stack-compaction.yaml configuration + And the active model has a registered context window And The service is restarted When I use "query" to ask question """ @@ -76,6 +74,7 @@ Feature: Conversation compaction Scenario: the native stream announces compaction on the query that crosses the threshold Given The service uses the lightspeed-stack-compaction.yaml configuration + And the active model has a registered context window And The service is restarted When I use "streaming_query" to ask question """ @@ -104,6 +103,7 @@ Feature: Conversation compaction Scenario: compaction stays off when disabled, even past the threshold Given The service uses the lightspeed-stack-compaction-disabled.yaml configuration + And the active model has a registered context window And The service is restarted When I use "query" to ask question """ diff --git a/tests/e2e/features/steps/README.md b/tests/e2e/features/steps/README.md index db2233cfe..885cfd273 100644 --- a/tests/e2e/features/steps/README.md +++ b/tests/e2e/features/steps/README.md @@ -20,6 +20,10 @@ Common steps for HTTP-related operations. Implementation of common test steps. +## [conversation_compaction.py](conversation_compaction.py) + +Steps observing conversation compaction from outside: `context_status`, the stream's `compaction` event, and the history the Conversations API keeps. + ## [feedback.py](feedback.py) Implementation of common test steps for the feedback API. diff --git a/tests/e2e/features/steps/conversation_compaction.py b/tests/e2e/features/steps/conversation_compaction.py new file mode 100644 index 000000000..691b2b01b --- /dev/null +++ b/tests/e2e/features/steps/conversation_compaction.py @@ -0,0 +1,157 @@ +"""Step definitions for the conversation-compaction e2e feature (LCORE-2230). + +Everything here observes compaction from outside the deployed stack — the +``context_status`` field on responses, the ``compaction`` event on the native +stream, and the conversation history the Conversations API serves. Steps +never import from or execute anything under ``src/`` +(``docs/testing/e2e_testing.md``, "Choosing the Test Layer"); the internals +(buffer, additive summaries, blocking) are integration tests (LCORE-1574). +""" + +import json +import os +from typing import Any, Optional + +import yaml +from behave import given, then # pyright: ignore +from behave.runner import Context + +from tests.e2e.utils.utils import absolute_repo_path, is_prow_environment + + +def _active_fixture_path(context: Context) -> str: + """Resolve the fixture file the last ``The service uses ...`` step applied. + + Mirrors the lookup in ``common.configure_service``: the configuration + directory from the Background, the deployment-mode subdirectory when it + exists, and the basename recorded on the context. The repo-root + ``lightspeed-stack.yaml`` copy is not used because on Prow the config is + pushed into a ConfigMap and that file is never written. + """ + config_name = context.active_lightspeed_stack_config_basename + mode_dir = "library-mode" if context.is_library_mode else "server-mode" + raw_base = getattr(context, "lightspeed_stack_config_directory", None) + base = str(raw_base).strip().rstrip("/") if raw_base else "tests/e2e/configuration" + mode_base = os.path.join(base, mode_dir) + if is_prow_environment(): + mode_base = absolute_repo_path(mode_base) + base = absolute_repo_path(base) + if os.path.isdir(mode_base): + return os.path.join(mode_base, config_name) + return os.path.join(base, config_name) + + +def _sse_events(response_text: str) -> list[dict[str, Any]]: + """Return the decoded SSE ``data:`` payloads of a streamed response, in order.""" + events: list[dict[str, Any]] = [] + for line in response_text.strip().split("\n"): + if not line.startswith("data: "): + continue + try: + events.append(json.loads(line[6:])) + except json.JSONDecodeError: + continue # Skip malformed lines + return events + + +def _first_index(events: list[dict[str, Any]], name: str) -> Optional[int]: + """Return the position of the first event called ``name``, or None.""" + return next((i for i, e in enumerate(events) if e.get("event") == name), None) + + +@given("the active model has a registered context window") +def require_context_window_for_active_model(context: Context) -> None: + """Skip the scenario when the active model has no ``context_windows`` entry. + + The compaction trigger only runs for models listed under + ``inference.context_windows`` in the active lightspeed-stack.yaml. The + fixtures list every fixed provider/model pair the e2e workflows use, but + the vLLM-backed runs (rhaiis, rhelai) take their model id from an env var + and mapping keys are not env-substituted, so those runs have no entry and + ``context_status`` can never become ``summarized``. Rather than fail there, + the scenario is skipped, the same way shield scenarios skip in library mode. + + Reads the fixture source file, never anything under ``src/``. The + provider/model pair comes from the context, which already reflects the + ``E2E_DEFAULT_*_OVERRIDE`` values on every path. + """ + fixture_path = _active_fixture_path(context) + with open(fixture_path, encoding="utf-8") as config_file: + config = yaml.safe_load(config_file) or {} + windows = (config.get("inference") or {}).get("context_windows") or {} + model_key = f"{context.default_provider}/{context.default_model}" + if model_key not in windows: + context.scenario.skip( + f"no context window registered for {model_key} in {fixture_path}; " + f"compaction cannot trigger (registered: {sorted(windows)})" + ) + + +@then('The response context_status is "{status}"') +def check_context_status(context: Context, status: str) -> None: + """Assert the non-streaming response reports the expected ``context_status`` (R7).""" + assert context.response is not None, "Request needs to be performed first" + response_json = context.response.json() + assert ( + "context_status" in response_json + ), f"context_status missing from response; keys: {list(response_json)}" + actual = response_json["context_status"] + assert actual == status, f"context_status is {actual!r}, expected {status!r}" + + +@then("The conversation history includes the following user queries") +def check_history_includes_user_queries(context: Context) -> None: + """Assert every listed query is still a user message in the conversation (R6). + + Reads ``chat_history`` from the GET conversation response and collects the + content of every ``user``-typed message across all turns; each row of the + scenario's "User query" table must appear among them verbatim. + """ + assert context.response is not None, "Request needs to be performed first" + assert context.table is not None, "Table with column 'User query' is required" + response_json = context.response.json() + assert "chat_history" in response_json, "chat_history not found in response" + user_queries = [ + message["content"].strip() + for turn in response_json["chat_history"] + for message in turn.get("messages", []) + if message.get("type") == "user" + ] + for row in context.table: + expected = row["User query"].strip() + assert expected in user_queries, ( + f"user query {expected!r} not found in conversation history; " + f"user queries present: {user_queries!r}" + ) + + +@then("The streamed response contains a compaction event before the first token") +def check_compaction_event_precedes_tokens(context: Context) -> None: + """Assert the stream announced compaction before any answer token (R12).""" + assert context.response is not None, "Request needs to be performed first" + events = _sse_events(context.response.text) + names = [e.get("event") for e in events] + compaction_at = _first_index(events, "compaction") + assert compaction_at is not None, f"no compaction event in stream; events: {names}" + token_at = _first_index(events, "token") + assert token_at is None or compaction_at < token_at, ( + f"compaction event at position {compaction_at} came after the first " + f"token at {token_at}; events: {names}" + ) + + +@then('The streamed response end event has context_status "{status}"') +def check_end_event_context_status(context: Context, status: str) -> None: + """Assert the stream's ``end`` event payload carries the expected ``context_status`` (R7).""" + assert context.response is not None, "Request needs to be performed first" + events = _sse_events(context.response.text) + end_at = _first_index(events, "end") + assert ( + end_at is not None + ), f"no end event in stream; events: {[e.get('event') for e in events]}" + data = events[end_at].get("data") or {} + assert "context_status" in data, f"end event carries no context_status: {data!r}" + actual = data["context_status"] + assert ( + actual == status + ), f"end event context_status is {actual!r}, expected {status!r}" From a3f7b56657de7a4d8adb1311c93b9d29160fbedb Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Mon, 14 Sep 2026 18:21:42 +0200 Subject: [PATCH 6/6] LCORE-2230: read the active config basename from module state in the gate The context-window gate step resolved the active fixture file from context.active_lightspeed_stack_config_basename. configure_service sets that attribute inside a step, so behave drops it when the scenario ends. When the next scenario asks for the YAML that is already active, configure_service returns early and never sets it again, and the gate passed None to os.path.join: TypeError: join() argument must be str, bytes, or os.PathLike object, not 'NoneType' The first two compaction scenarios both use lightspeed-stack-compaction.yaml, so the streaming scenario errored on the Konflux 0-8 e2e run while the other two passed. The GitHub e2e workflow runs on pull_request_target with main's tag expression, which does not select @cfg_compaction yet, so it never ran the feature. The gate now reads get_active_lightspeed_stack_config_basename(), the module-level state configure_service keeps across scenarios, and asserts a configuration was applied instead of failing inside os.path.join. behave loads environment.py first, which imports steps.common as a regular module, so the registered configure_service and this getter share the same state (checked against the step registry). --- .../features/steps/conversation_compaction.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/e2e/features/steps/conversation_compaction.py b/tests/e2e/features/steps/conversation_compaction.py index 691b2b01b..777d454a7 100644 --- a/tests/e2e/features/steps/conversation_compaction.py +++ b/tests/e2e/features/steps/conversation_compaction.py @@ -16,6 +16,7 @@ from behave import given, then # pyright: ignore from behave.runner import Context +from tests.e2e.features.steps.common import get_active_lightspeed_stack_config_basename from tests.e2e.utils.utils import absolute_repo_path, is_prow_environment @@ -24,11 +25,19 @@ def _active_fixture_path(context: Context) -> str: Mirrors the lookup in ``common.configure_service``: the configuration directory from the Background, the deployment-mode subdirectory when it - exists, and the basename recorded on the context. The repo-root - ``lightspeed-stack.yaml`` copy is not used because on Prow the config is - pushed into a ConfigMap and that file is never written. + exists, and the active basename. The basename is read from the module + state in ``common``, not from the context: behave drops attributes a step + sets once the scenario ends, and ``configure_service`` returns early + without setting them again when a scenario asks for the YAML that is + already active. The repo-root ``lightspeed-stack.yaml`` copy is not used + because on Prow the config is pushed into a ConfigMap and that file is + never written. """ - config_name = context.active_lightspeed_stack_config_basename + config_name = get_active_lightspeed_stack_config_basename() + assert config_name is not None, ( + "no lightspeed-stack configuration applied yet; " + "run 'The service uses the ... configuration' first" + ) mode_dir = "library-mode" if context.is_library_mode else "server-mode" raw_base = getattr(context, "lightspeed_stack_config_directory", None) base = str(raw_base).strip().rstrip("/") if raw_base else "tests/e2e/configuration"