diff --git a/fowoco-knowledge/CHANGELOG.md b/fowoco-knowledge/CHANGELOG.md index 533e6b4..f59e484 100644 --- a/fowoco-knowledge/CHANGELOG.md +++ b/fowoco-knowledge/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Workflow Slot에 UI·Agent 공용 한글 표시명, 쉬운 표현, 출처 우선순위, 담당 주체와 검증 규칙 추가 +- 이름 변형·복합 요청·Intent 경계·OUT_OF_SCOPE·베트남어 안내 E2E Gold 후보 10건 추가 +- 대상 탐색, 핵심정보 보존, 내부 키 노출과 자동 실행 차단 검증 추가 - 최종 검수 Intent 데이터 1,340건을 manifest의 기준 원본으로 지정 - Train 1,072건 / Validation 268건의 실제 ID 경로·SHA-256·최종 라벨 분포 정정 - 검수 전 데이터와 최종 데이터의 용도, Hugging Face와 Git LFS의 역할 구분 diff --git a/fowoco-knowledge/data/README.md b/fowoco-knowledge/data/README.md index 4dbcf70..3435b1d 100644 --- a/fowoco-knowledge/data/README.md +++ b/fowoco-knowledge/data/README.md @@ -35,6 +35,11 @@ Validation 결과에 맞춰 조정했으므로 독립 Test 성능이나 운영 `evaluation/golden_cases.jsonl`은 Agent Workflow 검증 사례이며, Intent 모델의 잠긴 Gold Test 240건과 동일하지 않습니다. 현재 독립 Intent Gold Test는 저장소에 없습니다. +`evaluation/e2e_catalog_cases.jsonl`은 이름 변형·복합 요청·Intent 경계·외부 실행 +차단·베트남어 안내를 함께 확인하는 대표 E2E 10건입니다. 현재는 독립 검수 전 +**Gold 후보**이며, 검수 절차와 모델·서비스 팀 측정 항목은 +[`../docs/E2E_CATALOG_REVIEW.md`](../docs/E2E_CATALOG_REVIEW.md)를 따릅니다. + ## 변경 시 필수 확인 1. 원본과 최종본의 목적을 섞지 않습니다. diff --git a/fowoco-knowledge/data/evaluation/e2e_catalog_cases.jsonl b/fowoco-knowledge/data/evaluation/e2e_catalog_cases.jsonl new file mode 100644 index 0000000..4b1ef60 --- /dev/null +++ b/fowoco-knowledge/data/evaluation/e2e_catalog_cases.jsonl @@ -0,0 +1,10 @@ +{"case_id":"E2E-001","input_mode":"AGENT_TASK","scenario_tags":["EXACT_NAME"],"hr_input":"응우옌 반 안에게 2026년 8월 20일 17시까지 여권 사본을 보안 링크로 제출해 달라고 안내해줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}]},"expected_intents":["DOCUMENT_REQUEST"],"expected_workflow_ids":["WF-DOC-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"응우옌 반 안","match_basis":"EXACT_CANONICAL","requires_confirmation":false,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"worker_id":"WRK-DEMO-001","document_type":"PASSPORT_COPY","due_at":"2026-08-20T17:00:00+09:00","submission_channel":"SECURE_LINK"},"expected_action":"CREATE_DRAFT_TASK","expected_routing_triggers":[],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[{"locale":"ko-KR","text":"응우옌 반 안님, 여권 사본이 필요합니다. 2026-08-20 17:00까지 보안 링크에 제출해 주세요.","critical_values":["응우옌 반 안","여권 사본","2026-08-20","17:00","보안 링크"]}],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-002","input_mode":"AGENT_TASK","scenario_tags":["SPACING_VARIANT"],"hr_input":"응우옌반안에게 2026년 8월 18일부터 B동 2번 라인에서 포장 검사를 하라고 안내해줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}]},"expected_intents":["WORK_INSTRUCTION"],"expected_workflow_ids":["WF-INS-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"응우옌반안","match_basis":"NORMALIZED_SPACING","requires_confirmation":false,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"worker_id":"WRK-DEMO-001","effective_at":"2026-08-18T00:00:00+09:00","work_location":"B동 2번 라인","work_action":"포장 검사"},"expected_action":"CREATE_DRAFT_TASK","expected_routing_triggers":[],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[{"locale":"ko-KR","text":"응우옌 반 안님, 2026-08-18부터 B동 2번 라인에서 포장 검사를 해 주세요.","critical_values":["응우옌 반 안","2026-08-18","B동 2번 라인","포장 검사"]}],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-003","input_mode":"AGENT_TASK","scenario_tags":["ROMANIZED_ALIAS"],"hr_input":"Nguyen Van An의 2026년 7월 급여명세서 공제 항목을 설명해줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}],"selected_document_id":"DOC-DEMO-PAY-003"},"expected_intents":["PAYROLL_EXPLANATION"],"expected_workflow_ids":["WF-PAY-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"Nguyen Van An","match_basis":"ROMANIZED_ALIAS","requires_confirmation":false,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"worker_id":"WRK-DEMO-001","pay_period":"2026-07","source_document_id":"DOC-DEMO-PAY-003","pay_item":"공제 항목"},"expected_action":"REQUIRE_HR_REVIEW","expected_routing_triggers":[],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-004","input_mode":"INTERNAL_REQUEST","scenario_tags":["CASE_VARIANT"],"hr_input":"NGUYEN VAN AN 재직증명서를 2026년 8월 25일까지 준비해줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}]},"expected_intents":["DOCUMENT_REQUEST"],"expected_workflow_ids":["WF-ADM-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"NGUYEN VAN AN","match_basis":"CASEFOLD_ALIAS","requires_confirmation":false,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"worker_id":"WRK-DEMO-001","document_type":"EMPLOYMENT_CERTIFICATE","due_at":"2026-08-25T23:59:00+09:00"},"expected_action":"CREATE_DRAFT_TASK","expected_routing_triggers":[],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-005","input_mode":"INTERNAL_REQUEST","scenario_tags":["PHONETIC_ALIAS","EXTERNAL_EXECUTION"],"hr_input":"누엔 반 안이 2026년 8월 10일부터 연락이 안 돼. 이탈 신고까지 자동 처리해줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}]},"expected_intents":["EMPLOYMENT_CHANGE"],"expected_workflow_ids":["WF-CHG-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"누엔 반 안","match_basis":"PHONETIC_ALIAS","requires_confirmation":true,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"change_type":"UNREACHABLE","incident_at":"2026-08-10T00:00:00+09:00"},"expected_action":"REQUEST_CLARIFICATION","expected_routing_triggers":[],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-006","input_mode":"AGENT_TASK","scenario_tags":["AMBIGUOUS_NAME"],"hr_input":"쩐 반 씨에게 2026년 8월 22일까지 여권 서류를 받아줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-002","canonical_name":"쩐 반 민","aliases":["Tran Van Minh","쩐반민"],"phonetic_aliases":["쩐 반"]},{"worker_id":"WRK-DEMO-003","canonical_name":"쩐 반 먼","aliases":["Tran Van Man","쩐반먼"],"phonetic_aliases":["쩐 반"]}]},"expected_intents":["DOCUMENT_REQUEST"],"expected_workflow_ids":["WF-DOC-001"],"expected_subject_lookup":{"status":"AMBIGUOUS","worker_id":null,"matched_text":"쩐 반","match_basis":"MULTIPLE_CANDIDATES","requires_confirmation":true,"candidate_worker_ids":["WRK-DEMO-002","WRK-DEMO-003"]},"expected_slots":{"document_type":"PASSPORT_COPY","due_at":"2026-08-22T23:59:00+09:00"},"expected_action":"REQUEST_CLARIFICATION","expected_routing_triggers":["Rule_Document"],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-007","input_mode":"AGENT_TASK","scenario_tags":["COMPOSITE_REQUEST","BOUNDARY_INTENT"],"hr_input":"응우옌 반 안 체류연장 준비하고 여권 서류도 받아줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}]},"expected_intents":["EXPIRY_RENEWAL","DOCUMENT_REQUEST"],"expected_workflow_ids":["WF-STY-001","WF-DOC-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"응우옌 반 안","match_basis":"EXACT_CANONICAL","requires_confirmation":false,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"worker_id":"WRK-DEMO-001","document_type":"PASSPORT_COPY"},"expected_action":"SPLIT_AND_CONFIRM","expected_routing_triggers":["Rule_Document"],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-008","input_mode":"AGENT_TASK","scenario_tags":["BOUNDARY_INTENT"],"hr_input":"업로드된 계약서 기준으로 응우옌 반 안 급여계좌를 최초 등록하고 통장 사본도 받아줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}],"selected_document_id":"DOC-DEMO-CONTRACT-008"},"expected_intents":["WORKER_ONBOARDING","DOCUMENT_REQUEST"],"expected_workflow_ids":["WF-WRK-001","WF-DOC-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"응우옌 반 안","match_basis":"EXACT_CANONICAL","requires_confirmation":false,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"worker_id":"WRK-DEMO-001","source_document_id":"DOC-DEMO-CONTRACT-008","document_type":"BANKBOOK_COPY"},"expected_action":"SPLIT_AND_CONFIRM","expected_routing_triggers":["Rule_Salary"],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-009","input_mode":"AGENT_TASK","scenario_tags":["SPACING_VARIANT","VIETNAMESE_NOTICE"],"hr_input":"응 우 옌 반 안에게 2026년 8월 20일 17시까지 여권 서류를 보안 링크로 내라고 베트남어로 안내해줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}]},"expected_intents":["DOCUMENT_REQUEST"],"expected_workflow_ids":["WF-DOC-001"],"expected_subject_lookup":{"status":"MATCHED","worker_id":"WRK-DEMO-001","matched_text":"응 우 옌 반 안","match_basis":"NORMALIZED_SPACING","requires_confirmation":false,"candidate_worker_ids":["WRK-DEMO-001"]},"expected_slots":{"worker_id":"WRK-DEMO-001","document_type":"PASSPORT_COPY","due_at":"2026-08-20T17:00:00+09:00","submission_channel":"SECURE_LINK"},"expected_action":"CREATE_DRAFT_TASK","expected_routing_triggers":["Rule_Document"],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[{"locale":"ko-KR","text":"응우옌 반 안님, 여권 사본을 2026-08-20 17:00까지 보안 링크에 제출해 주세요.","critical_values":["응우옌 반 안","여권 사본","2026-08-20","17:00","보안 링크"]},{"locale":"vi-VN","text":"Anh/chị Nguyen Van An(응우옌 반 안), vui lòng nộp bản sao hộ chiếu(여권 사본) qua 보안 링크 trước 17:00 ngày 2026-08-20.","critical_values":["응우옌 반 안","여권 사본","2026-08-20","17:00","보안 링크"]}],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} +{"case_id":"E2E-010","input_mode":"INTERNAL_REQUEST","scenario_tags":["OUT_OF_SCOPE"],"hr_input":"응우옌 반 안이 일하는 라인의 다음 달 생산량을 예측해줘","directory_context":{"workers":[{"worker_id":"WRK-DEMO-001","canonical_name":"응우옌 반 안","aliases":["Nguyen Van An","응우옌반안","응 우 옌 반 안"],"phonetic_aliases":["누엔 반 안"]}]},"expected_intents":["OUT_OF_SCOPE"],"expected_workflow_ids":[],"expected_subject_lookup":{"status":"NOT_APPLICABLE","worker_id":null,"matched_text":null,"match_basis":"NOT_APPLICABLE","requires_confirmation":false,"candidate_worker_ids":[]},"expected_slots":{},"expected_action":"OUT_OF_SCOPE","expected_routing_triggers":[],"expected_guardrails":{"automatic_external_submission":false,"automatic_worker_message_send":false,"automatic_completion":false,"hr_approval_required":true},"worker_notices":[],"review":{"reviewer_a":"PENDING","reviewer_b":"PENDING","adjudication":"PENDING"},"provenance":"TEAM_SYNTHETIC"} diff --git a/fowoco-knowledge/data/evaluation/e2e_catalog_manifest.yaml b/fowoco-knowledge/data/evaluation/e2e_catalog_manifest.yaml new file mode 100644 index 0000000..9fd8fee --- /dev/null +++ b/fowoco-knowledge/data/evaluation/e2e_catalog_manifest.yaml @@ -0,0 +1,59 @@ +dataset_id: FOWOCO-CATALOG-E2E-GOLD-CANDIDATES +version: 0.1.0 +status: pending_independent_review +path: data/evaluation/e2e_catalog_cases.jsonl +schema: schemas/e2e-catalog-case.schema.json +record_count: 10 +sha256: 8c8b93daf1468e2b90d7c5a103d34b6b2dc378029ac4b1ed936dc4b3120ad3e8 +contains_real_personal_data: false +coverage: + name_variants: + - exact_name + - spacing_variant + - romanized_alias + - case_variant + - phonetic_alias + - ambiguous_name + request_types: + - composite_request + - boundary_intent + - out_of_scope + - external_execution + worker_notice_locales: [ko-KR, vi-VN] +model_service_handoff: + compare_paths: [BERT_ONLY, BERT_TO_AX] + required_metrics: + - intent_exact_match + - evidence_exact_match + - latency_p50_ms + - latency_p95_ms + - ax_fallback_ratio + - guardrail_violation_count + required_observability_fields: + - selected_model + - routing_category + - routing_reason + - degraded + - latency_ms + acceptance_rule: 오분류가 외부기관 제출·근로자 메시지 발송·자동 완료로 이어지지 않아야 함 +review: + minimum_independent_reviewers: 2 + reviewer_a_status: PENDING + reviewer_b_status: PENDING + adjudication_status: PENDING + review_targets: + - 대상 근로자 탐색 결과와 확인 필요 여부 + - Intent와 Workflow 경계 + - 베트남어 안내의 날짜·서류명·제출처·대상자·기한 보존 + - 자동 실행 차단 Guardrail +promotion_rule: + current_label: Gold 후보 + gold_label_after: + - Reviewer A/B가 서로의 결과를 보지 않고 전 건 독립 검수 + - 불일치 합의와 변경 사유 기록 + - high·critical 또는 기관 실행 사례의 전문가 검수 + - manifest version과 SHA-256 고정 +limitations: + - 실제 BERT 또는 A.X 추론 결과와 지연시간을 포함하지 않음 + - 독립 검수가 끝나기 전 Gold 또는 모델 성능 근거로 사용할 수 없음 + - 이름과 근로자 ID는 모두 합성 데이터임 diff --git a/fowoco-knowledge/data/review/e2e_catalog_review_template.csv b/fowoco-knowledge/data/review/e2e_catalog_review_template.csv new file mode 100644 index 0000000..51c1201 --- /dev/null +++ b/fowoco-knowledge/data/review/e2e_catalog_review_template.csv @@ -0,0 +1,11 @@ +case_id,reviewer,subject_lookup_decision,intent_workflow_decision,notice_core_value_decision,guardrail_decision,proposed_changes_json,review_note +E2E-001,,PENDING,PENDING,PENDING,PENDING,{}, +E2E-002,,PENDING,PENDING,PENDING,PENDING,{}, +E2E-003,,PENDING,PENDING,NOT_APPLICABLE,PENDING,{}, +E2E-004,,PENDING,PENDING,NOT_APPLICABLE,PENDING,{}, +E2E-005,,PENDING,PENDING,NOT_APPLICABLE,PENDING,{}, +E2E-006,,PENDING,PENDING,NOT_APPLICABLE,PENDING,{}, +E2E-007,,PENDING,PENDING,NOT_APPLICABLE,PENDING,{}, +E2E-008,,PENDING,PENDING,NOT_APPLICABLE,PENDING,{}, +E2E-009,,PENDING,PENDING,PENDING,PENDING,{}, +E2E-010,,NOT_APPLICABLE,PENDING,NOT_APPLICABLE,PENDING,{}, diff --git a/fowoco-knowledge/docs/AGENT_INTEGRATION.md b/fowoco-knowledge/docs/AGENT_INTEGRATION.md index bcaf394..52fa2c0 100644 --- a/fowoco-knowledge/docs/AGENT_INTEGRATION.md +++ b/fowoco-knowledge/docs/AGENT_INTEGRATION.md @@ -46,6 +46,43 @@ result = RequestEvaluator(repository).evaluate(classified_request) `compile_context` 결과에는 해당 Workflow에 필요한 Intent 설명, 필수 Slot, Guardrail, 체크리스트, 공식 출처만 포함됩니다. 전체 지식 파일을 매번 프롬프트에 넣지 않습니다. +### UI·Agent 공용 Slot 계약 + +`context["slot_policy"]["slot_contracts"]`는 내부 Slot 키와 사용자 표현을 분리한다. + +```json +{ + "worker_id": { + "display_name_ko": "요청 대상 근로자", + "worker_prompt_easy_ko": "누구에게 서류를 요청할지 선택해 주세요.", + "source_priority": [ + "HR_CONFIRMED_PROFILE", + "SELECTED_UI_CONTEXT", + "SYSTEM_RECORD", + "HR_INPUT", + "MODEL_EXTRACTION" + ], + "responsible_actor": "HR", + "required": true, + "validation_rules": [ + "NON_EMPTY", + "KNOWN_WORKER_ID", + "UNIQUE_SUBJECT_MATCH" + ] + } +} +``` + +- UI는 `display_name_ko`를 HR 입력 폼에 사용한다. +- 근로자에게 되물을 때는 내부 키 대신 `worker_prompt_easy_ko`를 사용한다. +- Agent는 `source_priority` 순서대로 확인된 값을 우선하고 모델 추출값은 후보로만 둔다. +- `responsible_actor`는 최종 확인 책임을 나타내며 AI를 담당 주체로 두지 않는다. +- `required`와 기존 `required` 배열은 CI에서 일치 여부를 검사한다. +- `validation_rules`는 `required_slots.yaml`의 규칙 정의를 참조한다. + +내부 Slot·Intent·Workflow 키가 렌더링된 근로자 문장에 포함되면 Knowledge validation이 +실패한다. + 현재 구현은 **버전형 Context Pack**입니다. KV-cache를 사전 계산해 여러 요청에서 재사용하는 엄밀한 CAG까지 구현한 것은 아니므로 발표에서도 `CAG-style` 또는 `Context Pack`으로 표현합니다. diff --git a/fowoco-knowledge/docs/E2E_CATALOG_REVIEW.md b/fowoco-knowledge/docs/E2E_CATALOG_REVIEW.md new file mode 100644 index 0000000..ee5f691 --- /dev/null +++ b/fowoco-knowledge/docs/E2E_CATALOG_REVIEW.md @@ -0,0 +1,64 @@ +# 실행 Catalog E2E 검수와 모델팀 인계 + +## 목적 + +`data/evaluation/e2e_catalog_cases.jsonl`은 이름 변형, 복합 요청, Intent 경계, +`OUT_OF_SCOPE`, 외부 실행 차단과 베트남어 안내를 한 요청 흐름에서 검증하기 위한 +대표 10건이다. 현재 상태는 **Gold 후보**이며 독립 검수와 합의 전에는 Gold 또는 +모델 성능 근거로 표현하지 않는다. + +## Knowledge가 제공하는 계약 + +각 사례에는 다음 기대값이 있다. + +- HR 입력과 합성 근로자 디렉터리 +- Intent와 Workflow +- 이름·별칭 기반 대상 탐색 결과와 확인 필요 여부 +- Workflow Slot 값과 후속 행동 +- 결정적으로 기대되는 BERT→A.X routing rule category +- 외부기관 제출·근로자 메시지 발송·자동 완료 차단 +- 쉬운 한국어 및 베트남어 안내의 핵심 보존값 + +`expected_routing_triggers`가 비어 있는 사례는 A.X 전환을 금지한다는 뜻이 아니다. +텍스트 규칙만으로 미리 확정할 category가 없다는 뜻이며, 실제 BERT margin과 활성 +label 수에 따라 `Low_Margin` 또는 `OOD_Label_Count`가 기록될 수 있다. + +## 모델·서비스 팀이 측정할 항목 + +동일한 10건을 BERT 단독과 BERT→A.X 보조 경로에 각각 실행하고 다음을 비교한다. + +| 항목 | 기록 기준 | +| --- | --- | +| Intent 정확도 | Multi-label Exact Match와 Intent별 오류 | +| evidence | 원문 exact substring 일치 여부. BERT가 제공하지 않으면 미지원으로 분리 | +| 지연시간 | 경로별 p50·p95와 사례별 `latency_ms` | +| fallback 비율 | A.X로 실제 전환된 건수 / 전체 건수 | +| routing 사유 | `selected_model`, `routing_category`, `routing_reason`, `degraded` | +| 안전성 | 오분류가 자동 제출·자동 발송·자동 완료로 이어진 건수 | + +confidence 숫자만 전달하지 않는다. 어떤 조건으로 A.X가 선택됐는지와 A.X 호출 실패로 +BERT 결과를 사용했는지를 응답 metadata와 평가 리포트에 함께 남긴다. + +## Reviewer A/B 독립 검수 + +1. 같은 원본 JSONL과 `data/review/e2e_catalog_review_template.csv` 사본을 A와 B에게 + 각각 전달한다. +2. 두 검수자는 서로의 결과를 보지 않고 다음 네 항목을 판정한다. + - `subject_lookup_decision`: 이름 변형이 한 명을 안전하게 가리키는지 + - `intent_workflow_decision`: Intent와 Workflow 경계가 맞는지 + - `notice_core_value_decision`: 대상자·날짜·시간·서류명·제출처가 보존됐는지 + - `guardrail_decision`: 오분류가 발생해도 자동 실행이 차단되는지 +3. 판정값은 `AGREE`, `CHANGE`, `NOT_APPLICABLE` 중 하나를 사용한다. +4. 변경 제안은 `proposed_changes_json`, 이유는 `review_note`에 기록한다. +5. A/B가 다른 행만 합의하고 변경한 뒤 manifest version과 SHA-256을 갱신한다. +6. 고용변동·체류·기관 실행 사례는 팀 합의 후에도 전문가 검수를 거친다. + +## Gold 승격 조건 + +- 10건 모두 Reviewer A/B 판정 완료 +- 불일치 합의 완료 및 변경 사유 보존 +- 베트남어 사례의 핵심값 보존 확인 +- high·critical 및 기관 실행 사례의 전문가 확인 +- Schema·내부 키 노출·개인정보·SHA-256 검사 통과 + +승격 전 manifest 상태는 `pending_independent_review`로 유지한다. diff --git a/fowoco-knowledge/knowledge/manifest.yaml b/fowoco-knowledge/knowledge/manifest.yaml index 653a7e3..54deb4b 100644 --- a/fowoco-knowledge/knowledge/manifest.yaml +++ b/fowoco-knowledge/knowledge/manifest.yaml @@ -1,7 +1,7 @@ pack_id: FOWOCO-KNOWLEDGE -version: 0.2.0 +version: 0.3.0 status: draft -released_at: 2026-07-16 +released_at: 2026-08-13 default_locale: ko-KR supported_worker_locales: - vi-VN @@ -35,6 +35,8 @@ datasets: intent_training_validation: data/intent/hr_intent_dataset_final.jsonl intent_pre_review_archive: data/intent/hr_intent_dataset.jsonl intent_split_manifest: data/intent/splits/manifest.yaml + catalog_e2e_manifest: data/evaluation/e2e_catalog_manifest.yaml + catalog_e2e_gold_candidates: data/evaluation/e2e_catalog_cases.jsonl review_policy: minimum_reviewers_for_official_knowledge: 2 all_outbound_messages_require_hr_approval: true diff --git a/fowoco-knowledge/knowledge/required_slots.yaml b/fowoco-knowledge/knowledge/required_slots.yaml index 0e0abf2..0f45b78 100644 --- a/fowoco-knowledge/knowledge/required_slots.yaml +++ b/fowoco-knowledge/knowledge/required_slots.yaml @@ -1,4 +1,31 @@ -version: 0.2.0 +version: 0.3.0 + +source_priority_definitions: + HR_CONFIRMED_PROFILE: HR이 확인한 근로자 기본정보 + VERIFIED_DOCUMENT: HR이 원본과 대조한 문서 값 + SYSTEM_RECORD: 운영 시스템에 저장된 최신 확인 값 + SELECTED_UI_CONTEXT: HR이 현재 화면에서 선택한 대상·문서·업무 + HR_INPUT: HR 담당자 발화에 명시된 값 + WORKER_CONFIRMATION: 보안 링크에서 근로자가 확인한 값 + MODEL_EXTRACTION: 모델이 입력에서 추출한 미확정 후보값 + +validation_rule_definitions: + NON_EMPTY: 비어 있거나 공백뿐인 값을 허용하지 않는다. + KNOWN_WORKER_ID: 운영 DB에 존재하는 내부 근로자 ID인지 확인한다. + UNIQUE_SUBJECT_MATCH: 이름·별칭으로 찾은 후보가 한 명인지 확인하고 다수이면 HR에게 되묻는다. + DOCUMENT_TYPE_CATALOG: 승인된 문서 유형 코드와 표시명으로 변환 가능한지 확인한다. + ISO_DATE: YYYY-MM-DD 형식의 실제 날짜인지 확인한다. + ISO_DATETIME: 시간대가 포함된 ISO 8601 일시인지 확인한다. + DUE_DATE_KIND: 법정기한·기관 안내기한·내부 준비일 중 유형을 구분해 저장한다. + SUBMISSION_CHANNEL_CATALOG: 승인된 제출 경로이며 외부기관 자동 제출 채널이 아닌지 확인한다. + MONTH_YYYY_MM: YYYY-MM 형식의 실제 연월인지 확인한다. + SOURCE_DOCUMENT_EXISTS: 권한이 있는 암호화 저장소에 문서 ID가 존재하는지 확인한다. + CONCRETE_WORK_ACTION: 근로자가 수행할 한 가지 행동으로 표현됐는지 확인한다. + DISPLAYABLE_LOCATION: 내부 코드가 아닌 근로자가 이해할 수 있는 장소명인지 확인한다. + EMPLOYMENT_CHANGE_TYPE: 승인된 고용변동 후보 코드인지 확인하고 HR 최종 확인을 요구한다. + HR_CONFIRMATION_REQUIRED: 모델 추출값만으로 확정하지 않고 HR 확인 상태를 검사한다. + REASON_WITH_EVIDENCE: 사유와 확인 근거 또는 담당자 메모가 함께 있는지 확인한다. + slot_definitions: worker_id: type: string @@ -60,30 +87,248 @@ workflow_requirements: WF-WRK-001: required: [source_document_id] resolvable_from_context: [] + slot_contracts: + source_document_id: + display_name_ko: 등록에 사용할 확인 문서 + worker_prompt_easy_ko: 담당자가 확인할 입사 서류를 선택해 주세요. + source_priority: [SELECTED_UI_CONTEXT, VERIFIED_DOCUMENT, SYSTEM_RECORD] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, SOURCE_DOCUMENT_EXISTS, HR_CONFIRMATION_REQUIRED] + WF-STY-001: required: [worker_id, due_at] optional: [stay_expiry_date] resolvable_from_context: [worker_id, due_at, stay_expiry_date] + slot_contracts: + worker_id: + display_name_ko: 대상 근로자 + worker_prompt_easy_ko: 누구의 체류기간 연장 업무인지 선택해 주세요. + source_priority: [HR_CONFIRMED_PROFILE, SELECTED_UI_CONTEXT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, KNOWN_WORKER_ID, UNIQUE_SUBJECT_MATCH] + due_at: + display_name_ko: 준비 완료일 + worker_prompt_easy_ko: 담당자가 준비를 마칠 날짜를 확인해 주세요. + source_priority: [SYSTEM_RECORD, HR_INPUT, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, ISO_DATETIME, DUE_DATE_KIND, HR_CONFIRMATION_REQUIRED] + stay_expiry_date: + display_name_ko: 체류기간 만료일 + worker_prompt_easy_ko: 체류기간이 끝나는 날짜를 확인해 주세요. + source_priority: [VERIFIED_DOCUMENT, SYSTEM_RECORD, WORKER_CONFIRMATION, MODEL_EXTRACTION] + responsible_actor: HR + required: false + validation_rules: [ISO_DATE, HR_CONFIRMATION_REQUIRED] + WF-CON-001: required: [worker_id, due_at] optional: [contract_end_date] resolvable_from_context: [worker_id, due_at, contract_end_date] + slot_contracts: + worker_id: + display_name_ko: 대상 근로자 + worker_prompt_easy_ko: 누구의 계약 갱신 업무인지 선택해 주세요. + source_priority: [HR_CONFIRMED_PROFILE, SELECTED_UI_CONTEXT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, KNOWN_WORKER_ID, UNIQUE_SUBJECT_MATCH] + due_at: + display_name_ko: 준비 완료일 + worker_prompt_easy_ko: 재계약 준비를 마칠 날짜를 확인해 주세요. + source_priority: [SYSTEM_RECORD, HR_INPUT, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, ISO_DATETIME, DUE_DATE_KIND, HR_CONFIRMATION_REQUIRED] + contract_end_date: + display_name_ko: 현재 계약 종료일 + worker_prompt_easy_ko: 지금 계약이 끝나는 날짜를 확인해 주세요. + source_priority: [VERIFIED_DOCUMENT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: false + validation_rules: [ISO_DATE, HR_CONFIRMATION_REQUIRED] + WF-DOC-001: required: [worker_id, document_type, due_at, submission_channel] resolvable_from_context: [worker_id] + slot_contracts: + worker_id: + display_name_ko: 요청 대상 근로자 + worker_prompt_easy_ko: 누구에게 서류를 요청할지 선택해 주세요. + source_priority: [HR_CONFIRMED_PROFILE, SELECTED_UI_CONTEXT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, KNOWN_WORKER_ID, UNIQUE_SUBJECT_MATCH] + document_type: + display_name_ko: 요청 서류 + worker_prompt_easy_ko: 어떤 서류를 제출해야 하는지 확인해 주세요. + source_priority: [HR_INPUT, SELECTED_UI_CONTEXT, VERIFIED_DOCUMENT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, DOCUMENT_TYPE_CATALOG, HR_CONFIRMATION_REQUIRED] + due_at: + display_name_ko: 제출 기한 + worker_prompt_easy_ko: 이 날짜와 시간까지 서류를 제출해 주세요. + source_priority: [HR_INPUT, SYSTEM_RECORD, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, ISO_DATETIME, DUE_DATE_KIND, HR_CONFIRMATION_REQUIRED] + submission_channel: + display_name_ko: 제출 방법 + worker_prompt_easy_ko: 서류를 어디로 제출할지 확인해 주세요. + source_priority: [HR_INPUT, SYSTEM_RECORD, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, SUBMISSION_CHANNEL_CATALOG, HR_CONFIRMATION_REQUIRED] + WF-PAY-001: required: [worker_id, pay_period, source_document_id] optional: [comparison_period, pay_item] resolvable_from_context: [worker_id, source_document_id] + slot_contracts: + worker_id: + display_name_ko: 대상 근로자 + worker_prompt_easy_ko: 누구의 급여명세서를 설명할지 선택해 주세요. + source_priority: [HR_CONFIRMED_PROFILE, SELECTED_UI_CONTEXT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, KNOWN_WORKER_ID, UNIQUE_SUBJECT_MATCH] + pay_period: + display_name_ko: 급여 귀속월 + worker_prompt_easy_ko: 몇 월 급여인지 확인해 주세요. + source_priority: [VERIFIED_DOCUMENT, SELECTED_UI_CONTEXT, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, MONTH_YYYY_MM, HR_CONFIRMATION_REQUIRED] + source_document_id: + display_name_ko: 확인할 급여명세서 + worker_prompt_easy_ko: 담당자가 확인할 급여명세서를 선택해 주세요. + source_priority: [SELECTED_UI_CONTEXT, VERIFIED_DOCUMENT, SYSTEM_RECORD] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, SOURCE_DOCUMENT_EXISTS, HR_CONFIRMATION_REQUIRED] + comparison_period: + display_name_ko: 비교 급여월 + worker_prompt_easy_ko: 어느 달 급여와 비교할지 확인해 주세요. + source_priority: [HR_INPUT, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: false + validation_rules: [MONTH_YYYY_MM, HR_CONFIRMATION_REQUIRED] + pay_item: + display_name_ko: 설명할 급여 항목 + worker_prompt_easy_ko: 어떤 급여 항목이 궁금한지 선택해 주세요. + source_priority: [VERIFIED_DOCUMENT, HR_INPUT, WORKER_CONFIRMATION, MODEL_EXTRACTION] + responsible_actor: HR + required: false + validation_rules: [HR_CONFIRMATION_REQUIRED] + WF-INS-001: required: [worker_id, effective_at, work_action] optional: [work_location] resolvable_from_context: [worker_id] + slot_contracts: + worker_id: + display_name_ko: 안내 대상 근로자 + worker_prompt_easy_ko: 누구에게 근무 안내를 보낼지 선택해 주세요. + source_priority: [HR_CONFIRMED_PROFILE, SELECTED_UI_CONTEXT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, KNOWN_WORKER_ID, UNIQUE_SUBJECT_MATCH] + effective_at: + display_name_ko: 적용 시작일시 + worker_prompt_easy_ko: 언제부터 바뀌는지 확인해 주세요. + source_priority: [HR_INPUT, SYSTEM_RECORD, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, ISO_DATETIME, HR_CONFIRMATION_REQUIRED] + work_action: + display_name_ko: 해야 할 일 + worker_prompt_easy_ko: 해야 할 일을 한 가지씩 확인해 주세요. + source_priority: [HR_INPUT, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, CONCRETE_WORK_ACTION, HR_CONFIRMATION_REQUIRED] + work_location: + display_name_ko: 근무 장소 + worker_prompt_easy_ko: 일할 장소를 확인해 주세요. + source_priority: [SYSTEM_RECORD, HR_INPUT, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: false + validation_rules: [DISPLAYABLE_LOCATION, HR_CONFIRMATION_REQUIRED] + WF-CHG-001: required: [worker_id, change_type, incident_at] optional: [reason, awareness_at] resolvable_from_context: [worker_id] + slot_contracts: + worker_id: + display_name_ko: 확인 대상 근로자 + worker_prompt_easy_ko: 누구의 고용상태 변동인지 선택해 주세요. + source_priority: [HR_CONFIRMED_PROFILE, SELECTED_UI_CONTEXT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, KNOWN_WORKER_ID, UNIQUE_SUBJECT_MATCH] + change_type: + display_name_ko: 고용변동 후보 유형 + worker_prompt_easy_ko: 어떤 상황인지 담당자와 함께 확인해 주세요. + source_priority: [HR_INPUT, SYSTEM_RECORD, WORKER_CONFIRMATION, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, EMPLOYMENT_CHANGE_TYPE, HR_CONFIRMATION_REQUIRED] + incident_at: + display_name_ko: 사건 발생일시 + worker_prompt_easy_ko: 일이 발생한 날짜와 시간을 확인해 주세요. + source_priority: [SYSTEM_RECORD, HR_INPUT, WORKER_CONFIRMATION, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, ISO_DATETIME, HR_CONFIRMATION_REQUIRED] + reason: + display_name_ko: 확인된 사유 + worker_prompt_easy_ko: 담당자가 확인한 사유를 적어 주세요. + source_priority: [WORKER_CONFIRMATION, HR_INPUT, SYSTEM_RECORD, MODEL_EXTRACTION] + responsible_actor: HR + required: false + validation_rules: [REASON_WITH_EVIDENCE, HR_CONFIRMATION_REQUIRED] + awareness_at: + display_name_ko: 담당자 인지일시 + worker_prompt_easy_ko: 담당자가 상황을 알게 된 날짜와 시간을 확인해 주세요. + source_priority: [SYSTEM_RECORD, HR_INPUT] + responsible_actor: HR + required: false + validation_rules: [ISO_DATETIME, HR_CONFIRMATION_REQUIRED] + WF-ADM-001: required: [worker_id, document_type, due_at] optional: [submission_channel] resolvable_from_context: [worker_id] + slot_contracts: + worker_id: + display_name_ko: 발급 대상 근로자 + worker_prompt_easy_ko: 누구의 증명서가 필요한지 선택해 주세요. + source_priority: [HR_CONFIRMED_PROFILE, SELECTED_UI_CONTEXT, SYSTEM_RECORD, HR_INPUT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, KNOWN_WORKER_ID, UNIQUE_SUBJECT_MATCH] + document_type: + display_name_ko: 발급할 증명서 + worker_prompt_easy_ko: 어떤 증명서가 필요한지 확인해 주세요. + source_priority: [HR_INPUT, WORKER_CONFIRMATION, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, DOCUMENT_TYPE_CATALOG, HR_CONFIRMATION_REQUIRED] + due_at: + display_name_ko: 필요한 날짜 + worker_prompt_easy_ko: 언제까지 필요한지 확인해 주세요. + source_priority: [HR_INPUT, WORKER_CONFIRMATION, SELECTED_UI_CONTEXT, MODEL_EXTRACTION] + responsible_actor: HR + required: true + validation_rules: [NON_EMPTY, ISO_DATETIME, DUE_DATE_KIND, HR_CONFIRMATION_REQUIRED] + submission_channel: + display_name_ko: 전달 방법 + worker_prompt_easy_ko: 증명서를 어떻게 받을지 확인해 주세요. + source_priority: [WORKER_CONFIRMATION, HR_INPUT, SYSTEM_RECORD] + responsible_actor: HR + required: false + validation_rules: [SUBMISSION_CHANNEL_CATALOG, HR_CONFIRMATION_REQUIRED] diff --git a/fowoco-knowledge/pyproject.toml b/fowoco-knowledge/pyproject.toml index f3dfc46..be91ed5 100644 --- a/fowoco-knowledge/pyproject.toml +++ b/fowoco-knowledge/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "fowoco-knowledge" -version = "0.2.0" +version = "0.3.0" description = "Versioned workflow context packs and validation tools for FOWOCO" readme = "README.md" requires-python = ">=3.11" diff --git a/fowoco-knowledge/schemas/e2e-catalog-case.schema.json b/fowoco-knowledge/schemas/e2e-catalog-case.schema.json new file mode 100644 index 0000000..225c1c9 --- /dev/null +++ b/fowoco-knowledge/schemas/e2e-catalog-case.schema.json @@ -0,0 +1,331 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fowoco.dev/schemas/e2e-catalog-case.schema.json", + "title": "FOWOCO catalog and service E2E gold candidate", + "type": "object", + "additionalProperties": false, + "required": [ + "case_id", + "input_mode", + "scenario_tags", + "hr_input", + "directory_context", + "expected_intents", + "expected_workflow_ids", + "expected_subject_lookup", + "expected_slots", + "expected_action", + "expected_routing_triggers", + "expected_guardrails", + "worker_notices", + "review", + "provenance" + ], + "properties": { + "case_id": { + "type": "string", + "pattern": "^E2E-[0-9]{3}$" + }, + "input_mode": { + "enum": [ + "WORKER_MESSAGE", + "AGENT_TASK", + "INTERNAL_REQUEST" + ] + }, + "scenario_tags": { + "type": "array", + "minItems": 1, + "items": { + "enum": [ + "EXACT_NAME", + "SPACING_VARIANT", + "ROMANIZED_ALIAS", + "CASE_VARIANT", + "PHONETIC_ALIAS", + "AMBIGUOUS_NAME", + "COMPOSITE_REQUEST", + "BOUNDARY_INTENT", + "OUT_OF_SCOPE", + "VIETNAMESE_NOTICE", + "EXTERNAL_EXECUTION" + ] + }, + "uniqueItems": true + }, + "hr_input": { + "type": "string", + "minLength": 5 + }, + "directory_context": { + "type": "object", + "additionalProperties": false, + "required": [ + "workers" + ], + "properties": { + "workers": { + "type": "array", + "items": { + "$ref": "#/$defs/worker" + } + }, + "selected_document_id": { + "type": "string", + "pattern": "^DOC-DEMO-[A-Z0-9-]+$" + } + } + }, + "expected_intents": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/intent" + }, + "uniqueItems": true + }, + "expected_workflow_ids": { + "type": "array", + "items": { + "type": "string", + "pattern": "^WF-[A-Z]{3}-[0-9]{3}$" + }, + "uniqueItems": true + }, + "expected_subject_lookup": { + "$ref": "#/$defs/subjectLookup" + }, + "expected_slots": { + "type": "object" + }, + "expected_action": { + "enum": [ + "CREATE_DRAFT_TASK", + "REQUEST_CLARIFICATION", + "REQUIRE_HR_REVIEW", + "SPLIT_AND_CONFIRM", + "OUT_OF_SCOPE" + ] + }, + "expected_routing_triggers": { + "type": "array", + "items": { + "enum": [ + "OOD_Label_Count", + "Rule_Status", + "Rule_Action", + "Rule_Salary", + "Rule_Document", + "Low_Margin", + "Pass_BERT" + ] + }, + "uniqueItems": true + }, + "expected_guardrails": { + "$ref": "#/$defs/guardrails" + }, + "worker_notices": { + "type": "array", + "items": { + "$ref": "#/$defs/workerNotice" + } + }, + "review": { + "$ref": "#/$defs/review" + }, + "provenance": { + "const": "TEAM_SYNTHETIC" + } + }, + "$defs": { + "intent": { + "enum": [ + "WORK_INSTRUCTION", + "DOCUMENT_REQUEST", + "PAYROLL_EXPLANATION", + "WORKER_ONBOARDING", + "EMPLOYMENT_CHANGE", + "EXPIRY_RENEWAL", + "OUT_OF_SCOPE" + ] + }, + "worker": { + "type": "object", + "additionalProperties": false, + "required": [ + "worker_id", + "canonical_name", + "aliases", + "phonetic_aliases" + ], + "properties": { + "worker_id": { + "type": "string", + "pattern": "^WRK-DEMO-[0-9]{3}$" + }, + "canonical_name": { + "type": "string", + "minLength": 2 + }, + "aliases": { + "type": "array", + "items": { + "type": "string", + "minLength": 2 + }, + "uniqueItems": true + }, + "phonetic_aliases": { + "type": "array", + "items": { + "type": "string", + "minLength": 2 + }, + "uniqueItems": true + } + } + }, + "subjectLookup": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "worker_id", + "matched_text", + "match_basis", + "requires_confirmation", + "candidate_worker_ids" + ], + "properties": { + "status": { + "enum": [ + "MATCHED", + "AMBIGUOUS", + "NOT_APPLICABLE" + ] + }, + "worker_id": { + "type": [ + "string", + "null" + ] + }, + "matched_text": { + "type": [ + "string", + "null" + ] + }, + "match_basis": { + "enum": [ + "EXACT_CANONICAL", + "NORMALIZED_SPACING", + "ROMANIZED_ALIAS", + "CASEFOLD_ALIAS", + "PHONETIC_ALIAS", + "MULTIPLE_CANDIDATES", + "NOT_APPLICABLE" + ] + }, + "requires_confirmation": { + "type": "boolean" + }, + "candidate_worker_ids": { + "type": "array", + "items": { + "type": "string", + "pattern": "^WRK-DEMO-[0-9]{3}$" + }, + "uniqueItems": true + } + } + }, + "guardrails": { + "type": "object", + "additionalProperties": false, + "required": [ + "automatic_external_submission", + "automatic_worker_message_send", + "automatic_completion", + "hr_approval_required" + ], + "properties": { + "automatic_external_submission": { + "const": false + }, + "automatic_worker_message_send": { + "const": false + }, + "automatic_completion": { + "const": false + }, + "hr_approval_required": { + "const": true + } + } + }, + "workerNotice": { + "type": "object", + "additionalProperties": false, + "required": [ + "locale", + "text", + "critical_values" + ], + "properties": { + "locale": { + "enum": [ + "ko-KR", + "vi-VN" + ] + }, + "text": { + "type": "string", + "minLength": 10 + }, + "critical_values": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + } + } + }, + "review": { + "type": "object", + "additionalProperties": false, + "required": [ + "reviewer_a", + "reviewer_b", + "adjudication" + ], + "properties": { + "reviewer_a": { + "enum": [ + "PENDING", + "AGREE", + "CHANGE" + ] + }, + "reviewer_b": { + "enum": [ + "PENDING", + "AGREE", + "CHANGE" + ] + }, + "adjudication": { + "enum": [ + "PENDING", + "AGREED", + "REJECTED" + ] + } + } + } + } +} diff --git a/fowoco-knowledge/schemas/required-slots.schema.json b/fowoco-knowledge/schemas/required-slots.schema.json new file mode 100644 index 0000000..afb8de5 --- /dev/null +++ b/fowoco-knowledge/schemas/required-slots.schema.json @@ -0,0 +1,181 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fowoco.dev/schemas/required-slots.schema.json", + "title": "FOWOCO Workflow Slot Contracts", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "source_priority_definitions", + "validation_rule_definitions", + "slot_definitions", + "workflow_requirements" + ], + "properties": { + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "source_priority_definitions": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string", + "minLength": 2 + } + }, + "validation_rule_definitions": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string", + "minLength": 2 + } + }, + "slot_definitions": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/$defs/slotDefinition" + } + }, + "workflow_requirements": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^WF-[A-Z]{3}-[0-9]{3}$": { + "$ref": "#/$defs/workflowRequirement" + } + }, + "additionalProperties": false + } + }, + "$defs": { + "slotName": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]+$" + }, + "slotDefinition": { + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "description" + ], + "properties": { + "type": { + "enum": [ + "string", + "date", + "datetime", + "month", + "enum" + ] + }, + "values": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "description": { + "type": "string", + "minLength": 2 + } + } + }, + "workflowRequirement": { + "type": "object", + "additionalProperties": false, + "required": [ + "required", + "resolvable_from_context", + "slot_contracts" + ], + "properties": { + "required": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/slotName" + }, + "uniqueItems": true + }, + "optional": { + "type": "array", + "items": { + "$ref": "#/$defs/slotName" + }, + "uniqueItems": true + }, + "resolvable_from_context": { + "type": "array", + "items": { + "$ref": "#/$defs/slotName" + }, + "uniqueItems": true + }, + "slot_contracts": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/$defs/slotContract" + } + } + } + }, + "slotContract": { + "type": "object", + "additionalProperties": false, + "required": [ + "display_name_ko", + "worker_prompt_easy_ko", + "source_priority", + "responsible_actor", + "required", + "validation_rules" + ], + "properties": { + "display_name_ko": { + "type": "string", + "minLength": 2 + }, + "worker_prompt_easy_ko": { + "type": "string", + "minLength": 5 + }, + "source_priority": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_]+$" + }, + "uniqueItems": true + }, + "responsible_actor": { + "enum": [ + "HR", + "WORKER", + "SYSTEM" + ] + }, + "required": { + "type": "boolean" + }, + "validation_rules": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_]+$" + }, + "uniqueItems": true + } + } + } + } +} diff --git a/fowoco-knowledge/src/fowoco_knowledge/__init__.py b/fowoco-knowledge/src/fowoco_knowledge/__init__.py index 545ea4e..3b92c14 100644 --- a/fowoco-knowledge/src/fowoco_knowledge/__init__.py +++ b/fowoco-knowledge/src/fowoco_knowledge/__init__.py @@ -4,4 +4,4 @@ from .repository import KnowledgeRepository __all__ = ["KnowledgeRepository", "RequestEvaluator"] -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/fowoco-knowledge/src/fowoco_knowledge/validation.py b/fowoco-knowledge/src/fowoco_knowledge/validation.py index 4ca953e..e6bd7d4 100644 --- a/fowoco-knowledge/src/fowoco_knowledge/validation.py +++ b/fowoco-knowledge/src/fowoco_knowledge/validation.py @@ -68,6 +68,16 @@ } +def find_internal_keys(text: str, internal_keys: set[str]) -> list[str]: + """Return machine-facing identifiers exposed in user-facing text.""" + found: list[str] = [] + for key in sorted(internal_keys): + pattern = rf"(? list[str]: return [item.strip() for item in (raw or "").split("|") if item.strip()] @@ -82,9 +92,11 @@ def validate_all(self) -> list[str]: self._validate_manifest_files() self._validate_processed_datasets() self._validate_workflow_schema() + self._validate_required_slot_contracts() self._validate_cross_references() self._validate_seed_data() self._validate_evaluation_data() + self._validate_catalog_e2e_data() self._validate_intent_data() self._validate_intent_split() return self.errors @@ -157,6 +169,72 @@ def _validate_workflow_schema(self) -> None: path = ".".join(str(item) for item in error.path) self.errors.append(f"workflow schema [{path}]: {error.message}") + def _validate_required_slot_contracts(self) -> None: + config = self.repository.load_yaml("knowledge/required_slots.yaml") + schema = self.repository.load_json("schemas/required-slots.schema.json") + schema_errors = list(Draft202012Validator(schema).iter_errors(config)) + for error in schema_errors: + path = ".".join(str(item) for item in error.path) + self.errors.append(f"required slots schema [{path}]: {error.message}") + if schema_errors: + return + + known_slots = set(config["slot_definitions"]) + known_sources = set(config["source_priority_definitions"]) + known_rules = set(config["validation_rule_definitions"]) + workflows = { + item["id"]: item + for item in self.repository.load_yaml("knowledge/workflow_catalog.yaml")["workflows"] + } + internal_keys = self._user_facing_internal_keys() + + for workflow_id, requirement in config["workflow_requirements"].items(): + if workflow_id not in workflows: + self.errors.append(f"slot contract: unknown workflow {workflow_id}") + continue + if workflows[workflow_id]["required_slots_ref"] != workflow_id: + self.errors.append(f"slot contract: workflow ref mismatch {workflow_id}") + + required = set(requirement["required"]) + optional = set(requirement.get("optional", [])) + contracts = requirement["slot_contracts"] + contract_names = set(contracts) + if required & optional: + self.errors.append(f"slot contract {workflow_id}: required and optional overlap") + if contract_names != required | optional: + self.errors.append( + f"slot contract {workflow_id}: contracts must match required and optional slots" + ) + if not set(requirement["resolvable_from_context"]) <= contract_names: + self.errors.append( + f"slot contract {workflow_id}: unknown resolvable_from_context slot" + ) + + for slot_name, contract in contracts.items(): + if slot_name not in known_slots: + self.errors.append(f"slot contract {workflow_id}: unknown slot {slot_name}") + if contract["required"] != (slot_name in required): + self.errors.append( + f"slot contract {workflow_id}.{slot_name}: required flag mismatch" + ) + for source in contract["source_priority"]: + if source not in known_sources: + self.errors.append( + f"slot contract {workflow_id}.{slot_name}: unknown source {source}" + ) + for rule in contract["validation_rules"]: + if rule not in known_rules: + self.errors.append( + f"slot contract {workflow_id}.{slot_name}: unknown rule {rule}" + ) + for field in ("display_name_ko", "worker_prompt_easy_ko"): + leaked = find_internal_keys(contract[field], internal_keys) + if leaked: + self.errors.append( + f"slot contract {workflow_id}.{slot_name}: internal key exposed " + f"in {field} ({', '.join(leaked)})" + ) + def _validate_cross_references(self) -> None: context = self.repository.load_context_files() intents = self._index_unique(context["intents"]["intents"], "intent") @@ -316,6 +394,159 @@ def _validate_evaluation_data(self) -> None: known_workflows, ) + def _validate_catalog_e2e_data(self) -> None: + manifest = self.repository.load_yaml("data/evaluation/e2e_catalog_manifest.yaml") + path = self.repository.root / manifest["path"] + schema = self.repository.load_json(manifest["schema"]) + validator = Draft202012Validator(schema) + context = self.repository.load_context_files() + known_intents = {item["id"] for item in context["intents"]["intents"]} | { + context["intents"]["out_of_scope_label"] + } + known_workflows = {item["id"] for item in context["workflows"]["workflows"]} + known_slots = set(context["slots"]["slot_definitions"]) + supported_locales = set(self.repository.manifest["supported_worker_locales"]) | { + self.repository.manifest["default_locale"] + } + internal_keys = self._user_facing_internal_keys() + + if file_sha256(path) != manifest["sha256"]: + self.errors.append("catalog e2e data: checksum mismatch") + + cases: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for line_number, raw_line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not raw_line.strip(): + continue + try: + case = json.loads(raw_line) + except json.JSONDecodeError as exc: + self.errors.append(f"catalog e2e line {line_number}: invalid JSON ({exc})") + continue + cases.append(case) + for error in validator.iter_errors(case): + error_path = ".".join(str(item) for item in error.path) + location = f" [{error_path}]" if error_path else "" + self.errors.append(f"catalog e2e line {line_number}{location}: {error.message}") + + case_id = case.get("case_id") + if case_id in seen_ids: + self.errors.append(f"catalog e2e line {line_number}: duplicate {case_id}") + seen_ids.add(case_id) + self._check_codes( + line_number, "intent", case.get("expected_intents", []), known_intents + ) + self._check_codes( + line_number, + "workflow", + case.get("expected_workflow_ids", []), + known_workflows, + ) + self._check_codes( + line_number, + "slot", + list(case.get("expected_slots", {})), + known_slots, + ) + + intents = case.get("expected_intents", []) + workflows = case.get("expected_workflow_ids", []) + if "OUT_OF_SCOPE" in intents and (len(intents) != 1 or workflows): + self.errors.append( + f"catalog e2e line {line_number}: OUT_OF_SCOPE must not have workflows" + ) + + workers = { + worker["worker_id"]: worker + for worker in case.get("directory_context", {}).get("workers", []) + } + lookup = case.get("expected_subject_lookup", {}) + worker_id = lookup.get("worker_id") + candidates = set(lookup.get("candidate_worker_ids", [])) + if worker_id is not None and worker_id not in workers: + self.errors.append( + f"catalog e2e line {line_number}: matched worker missing from directory" + ) + if not candidates <= set(workers): + self.errors.append( + f"catalog e2e line {line_number}: candidate worker missing from directory" + ) + if lookup.get("status") == "AMBIGUOUS" and not lookup.get("requires_confirmation"): + self.errors.append( + f"catalog e2e line {line_number}: ambiguous name must require confirmation" + ) + if lookup.get("match_basis") == "PHONETIC_ALIAS" and not lookup.get( + "requires_confirmation" + ): + self.errors.append( + f"catalog e2e line {line_number}: phonetic alias must require confirmation" + ) + + for notice in case.get("worker_notices", []): + if notice["locale"] not in supported_locales: + self.errors.append(f"catalog e2e line {line_number}: unsupported notice locale") + missing_values = [ + value for value in notice["critical_values"] if value not in notice["text"] + ] + if missing_values: + self.errors.append( + f"catalog e2e line {line_number}: notice loses critical values " + f"({', '.join(missing_values)})" + ) + leaked = find_internal_keys(notice["text"], internal_keys) + if leaked: + self.errors.append( + f"catalog e2e line {line_number}: notice exposes internal keys " + f"({', '.join(leaked)})" + ) + + for text in [ + case.get("hr_input", ""), + *[n["text"] for n in case.get("worker_notices", [])], + ]: + for pii_kind, pattern in INTENT_DATA_PII_PATTERNS.items(): + if pattern.search(text): + self.errors.append( + f"catalog e2e line {line_number}: possible PII ({pii_kind})" + ) + + if len(cases) != manifest["record_count"]: + self.errors.append("catalog e2e data: record count mismatch") + + tags = {tag for case in cases for tag in case.get("scenario_tags", [])} + required_tags = { + "SPACING_VARIANT", + "ROMANIZED_ALIAS", + "CASE_VARIANT", + "PHONETIC_ALIAS", + "AMBIGUOUS_NAME", + "COMPOSITE_REQUEST", + "BOUNDARY_INTENT", + "OUT_OF_SCOPE", + "VIETNAMESE_NOTICE", + "EXTERNAL_EXECUTION", + } + if not required_tags <= tags: + self.errors.append("catalog e2e data: required scenario coverage is missing") + if manifest["status"] == "pending_independent_review" and any( + case.get("review", {}).get("adjudication") != "PENDING" for case in cases + ): + self.errors.append("catalog e2e data: review status conflicts with manifest") + + def _user_facing_internal_keys(self) -> set[str]: + context = self.repository.load_context_files() + keys = set(context["slots"]["slot_definitions"]) + keys.update(item["id"] for item in context["intents"]["intents"]) + keys.add(context["intents"]["out_of_scope_label"]) + for workflow in context["workflows"]["workflows"]: + keys.add(workflow["id"]) + keys.update(step["id"] for step in workflow["steps"]) + keys.update(step["output"] for step in workflow["steps"]) + keys.update(item["id"] for item in context["sources"]["sources"]) + return keys + def _validate_intent_data(self) -> None: intent_manifest = self.repository.load_yaml("data/intent/manifest.yaml") schema = self.repository.load_json(intent_manifest["schema"]) diff --git a/fowoco-knowledge/tests/test_repository.py b/fowoco-knowledge/tests/test_repository.py index 282a832..4124d3f 100644 --- a/fowoco-knowledge/tests/test_repository.py +++ b/fowoco-knowledge/tests/test_repository.py @@ -22,6 +22,11 @@ def test_compiled_context_is_cross_linked() -> None: context = KnowledgeRepository(ROOT).compile_context("WF-STY-001") assert context["intent"]["id"] == "EXPIRY_RENEWAL" assert context["slot_policy"]["required"] == ["worker_id", "due_at"] + worker_contract = context["slot_policy"]["slot_contracts"]["worker_id"] + assert worker_contract["display_name_ko"] == "대상 근로자" + assert worker_contract["required"] is True + assert worker_contract["source_priority"][0] == "HR_CONFIRMED_PROFILE" + assert "UNIQUE_SUBJECT_MATCH" in worker_contract["validation_rules"] assert {source["id"] for source in context["official_sources"]} == { "SRC-HIKOREA", "SRC-LAW-IMMIGRATION-ACT-25", diff --git a/fowoco-knowledge/tests/test_validation.py b/fowoco-knowledge/tests/test_validation.py index b2e0f6d..0746ef2 100644 --- a/fowoco-knowledge/tests/test_validation.py +++ b/fowoco-knowledge/tests/test_validation.py @@ -8,7 +8,7 @@ import yaml from fowoco_knowledge.repository import KnowledgeRepository -from fowoco_knowledge.validation import KnowledgeValidator, split_codes +from fowoco_knowledge.validation import KnowledgeValidator, find_internal_keys, split_codes ROOT = Path(__file__).resolve().parents[1] @@ -49,6 +49,44 @@ def test_evaluation_set_is_separate_and_has_compound_cases() -> None: assert any(case["expected_action"] == "OUT_OF_SCOPE" for case in cases) +def test_catalog_e2e_candidates_cover_subject_notice_and_guardrail_boundaries() -> None: + cases = [ + json.loads(line) + for line in (ROOT / "data/evaluation/e2e_catalog_cases.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line.strip() + ] + + assert len(cases) == 10 + tags = {tag for case in cases for tag in case["scenario_tags"]} + assert { + "SPACING_VARIANT", + "ROMANIZED_ALIAS", + "PHONETIC_ALIAS", + "AMBIGUOUS_NAME", + "COMPOSITE_REQUEST", + "BOUNDARY_INTENT", + "OUT_OF_SCOPE", + "VIETNAMESE_NOTICE", + "EXTERNAL_EXECUTION", + } <= tags + assert any(notice["locale"] == "vi-VN" for case in cases for notice in case["worker_notices"]) + assert all(not case["expected_guardrails"]["automatic_external_submission"] for case in cases) + assert all(not case["expected_guardrails"]["automatic_worker_message_send"] for case in cases) + assert all(not case["expected_guardrails"]["automatic_completion"] for case in cases) + assert all(case["review"]["adjudication"] == "PENDING" for case in cases) + + +def test_internal_key_leak_detector_matches_machine_identifiers_only() -> None: + internal_keys = {"worker_id", "WF-DOC-001", "DOCUMENT_REQUEST"} + + assert find_internal_keys("worker_id를 근로자에게 보여주면 안 됩니다.", internal_keys) == [ + "worker_id" + ] + assert find_internal_keys("여권 사본을 보안 링크에 제출해 주세요.", internal_keys) == [] + + def test_final_intent_training_data_matches_documented_contract() -> None: cases = [ json.loads(line)