From 91c1a8a0cc820e45698a62ab8d015b8a2e3e6060 Mon Sep 17 00:00:00 2001 From: wooh Date: Sun, 26 Jul 2026 03:12:38 +0900 Subject: [PATCH 1/2] =?UTF-8?q?[Feat]=20=EC=9E=90=EA=B8=B0=EC=86=8C?= =?UTF-8?q?=EA=B0=9C=EC=84=9C=20=EB=B6=84=EC=84=9D=20Hybrid=20Exact=20?= =?UTF-8?q?=EA=B8=B0=EC=A4=80=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - analysis.mode=hybrid-exact 실행 모드 추가 - Hybrid Exact에서 기존 single-pass 전체 분석 경로 실행 - Hybrid Exact에서 기존 two-pass 전체 분석 경로 실행 - questionAnalyses, 점수, feedback, strengths는 single-pass 결과 사용 - missingKeywords는 two-pass 전체 결과 사용 - 기존 missingKeywords 전용 Hybrid 최적화 경로 제거 - Hybrid Exact merge 시 새 AnalysisLlmResponse 생성으로 필드 출처 명시 - analysis-eval 전용 offline hybrid merge Runner 추가 - v5-A CSV와 two-pass CSV를 caseId 기준으로 exact merge - CSV caseId mismatch, duplicate, 필수 JSON 파싱 실패 검증 추가 - analysis/nlg-judge/hybrid-merge Runner 동시 실행 방지 - Hybrid Exact merge 및 Runner 조건 테스트 추가 --- ...evaluation_nlg_judge_hybrid_comparison.csv | 4 + evaluation/evaluation_nlg_judge_hybrid_v1.csv | 21 ++ .../evaluation/EvaluationAnalysisRunner.java | 7 +- .../evaluation/EvaluationCsvSupport.java | 18 ++ .../EvaluationRunnerFlagValidator.java | 11 +- .../evaluation/HybridExactMergeRunner.java | 136 ++++++++++++ .../evaluation/HybridExactMergeService.java | 171 +++++++++++++++ .../evaluation/NlgEvaluationRunner.java | 7 +- .../analysis/service/AnalysisAiClient.java | 161 ++++++++++++-- .../resources/application-analysis-eval.yaml | 5 + .../EvaluationAnalysisRunnerSafetyTest.java | 64 +++++- .../HybridExactMergeServiceTest.java | 203 ++++++++++++++++++ .../service/AnalysisAiClientTest.java | 121 +++++++++++ 13 files changed, 901 insertions(+), 28 deletions(-) create mode 100644 evaluation/evaluation_nlg_judge_hybrid_comparison.csv create mode 100644 evaluation/evaluation_nlg_judge_hybrid_v1.csv create mode 100644 src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeRunner.java create mode 100644 src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java create mode 100644 src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java diff --git a/evaluation/evaluation_nlg_judge_hybrid_comparison.csv b/evaluation/evaluation_nlg_judge_hybrid_comparison.csv new file mode 100644 index 00000000..bbe43906 --- /dev/null +++ b/evaluation/evaluation_nlg_judge_hybrid_comparison.csv @@ -0,0 +1,4 @@ +sourceResultFile,caseCount,successCount,averageRelevance,averageProblemValidity,averageSentenceTypeConsistency,averageReasonCorrectness,averageContextAwareness,averageFaithfulness,averageTenseConsistency,averageUsability,averageNonMeta,averageMeaningPreservation,noAnalysisAppropriateness,strengthsPrecision,strengthsCoverage,missingKeywordsPrecision,missingKeywordsCoverage,overallUsefulness,averageJudgeInputTokens,averageJudgeOutputTokens,averageJudgeLatencyMs,averageAnalysisCount,metaImprovementRate,unsupportedFactRate,falsePositiveAnalysisRate,fatalErrorRate,errorCodeCounts +evaluation/evaluation_nlg_judge_v5a_v2.csv,20,20,3.06,2.75,3.75,3.19,2.06,4.0,4.13,3.81,3.69,4.06,2.05,4.45,4.45,3.45,3.45,2.75,,,4214.25,0.8,0.0,0.0,0.0,0.0,"{""CONTEXT_IGNORED"":4,""MISSED_ANALYSIS"":18,""MISSED_MISSING_KEYWORD"":10,""NONE"":1}" +evaluation/evaluation_nlg_judge_two_pass_v2_1_v2.csv,20,20,3.33,2.5,3.5,3.33,2.17,4.0,4.17,3.5,3.33,4.0,1.85,4.14,4.14,4.07,4.07,2.6,,,3821.65,0.45,0.0,0.0,0.0,0.0,"{""CONTEXT_IGNORED"":2,""MISSED_ANALYSIS"":19,""MISSED_MISSING_KEYWORD"":2,""MISSED_STRENGTH"":2,""NONE"":1}" +evaluation/evaluation_nlg_judge_hybrid_v1.csv,20,20,3.17,2.75,3.58,3.17,2.0,3.92,4.08,3.67,4.0,4.08,2.0,3.55,3.55,1.95,1.95,2.1,3617.55,235.75,3506.55,0.65,0.0,0.0,0.0,0.0,"{""CONTEXT_IGNORED"":1,""MISSED_ANALYSIS"":20,""MISSED_MISSING_KEYWORD"":20}" diff --git a/evaluation/evaluation_nlg_judge_hybrid_v1.csv b/evaluation/evaluation_nlg_judge_hybrid_v1.csv new file mode 100644 index 00000000..0b6fd492 --- /dev/null +++ b/evaluation/evaluation_nlg_judge_hybrid_v1.csv @@ -0,0 +1,21 @@ +caseId,sourceResultFile,analysisCount,averageRelevance,averageProblemValidity,averageSentenceTypeConsistency,averageReasonCorrectness,averageContextAwareness,averageFaithfulness,averageTenseConsistency,averageUsability,averageNonMeta,averageMeaningPreservation,noAnalysisAppropriateness,strengthsPrecision,strengthsCoverage,missingKeywordsPrecision,missingKeywordsCoverage,overallUsefulness,errorCodes,shortRationale,judgeInputTokens,judgeOutputTokens,judgeLatencyMs,failureStage +EV-01,evaluation/evaluation_ai_results_hybrid_v1.csv,0,,,,,,,,,,,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","답변에 명백한 문제 문장이 없으나, 분석이 비어 있어 중요한 첨삭 대상을 놓쳤습니다. 또한, JD의 핵심 경험 요구사항이 누락되었습니다.",3642,153,3460, +EV-02,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,3.0,4.0,3.0,2.0,4.0,4.0,4.0,4.0,4.0,2,4,4,1,1,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 경험이 잘 드러나지 않은 부분이 있으며, JD의 핵심 키워드가 누락되었습니다. 전반적으로 개선이 필요합니다.",2711,289,3483, +EV-03,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,3.0,4.0,3.0,2.0,4.0,4.0,3.0,4.0,4.0,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","분석에서 중요한 경험과 강점이 누락되었으며, 명백한 문제 문장이 존재하지만 분석이 부족하여 전반적인 유용성이 낮게 평가되었습니다.",4208,287,4661, +EV-04,evaluation/evaluation_ai_results_hybrid_v1.csv,0,,,,,,,,,,,2,3,3,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","분석이 전혀 이루어지지 않았고, 중요한 첨삭 대상을 놓쳤습니다. 또한, JD의 핵심 경험 요구사항이 누락되었습니다.",4793,147,2841, +EV-05,evaluation/evaluation_ai_results_hybrid_v1.csv,1,4.0,3.0,4.0,3.0,2.0,5.0,5.0,4.0,5.0,5.0,2,3,3,2,2,3,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 포부는 명확하나 구체적인 실행 방법이 부족하며, JD의 핵심 경험 요구사항이 누락되었습니다. 전반적으로 유용성은 중간 수준입니다.",3120,301,3162, +EV-06,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,3.0,4.0,3.0,2.0,4.0,4.0,3.0,4.0,4.0,2,3,3,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","자기소개서에서 주요 경험과 강점이 잘 드러나지 않았고, 중요한 키워드가 누락되었습니다. 분석이 부족하여 전반적인 유용성이 낮게 평가되었습니다.",5796,290,3698, +EV-07,evaluation/evaluation_ai_results_hybrid_v1.csv,0,,,,,,,,,,,2,5,5,1,1,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","분석이 전혀 이루어지지 않았고, 중요한 경험 키워드가 누락되었습니다. 전반적으로 개선이 필요합니다.",2545,139,1994, +EV-08,evaluation/evaluation_ai_results_hybrid_v1.csv,0,,,,,,,,,,,2,3,3,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 경험이 JD의 주요 요구사항을 충분히 반영하지 못하고 있으며, 분석이 전혀 이루어지지 않아 중요한 문제를 놓쳤습니다.",4207,144,2814, +EV-09,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,3.0,4.0,3.0,2.0,4.0,4.0,4.0,4.0,4.0,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 경험은 유용하지만, JD의 주요 요구사항인 부가세 및 법인세 신고 실무 경험과 결산 및 관리회계 실무 경험이 누락되었습니다. 또한, 분석이 부족하여 중요한 문제를 놓쳤습니다.",3584,324,4247, +EV-10,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,3.0,3.0,3.0,2.0,4.0,4.0,4.0,4.0,4.0,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원서에서 주요 업무와 관련된 경험이 부족하며, 분석이 결여된 부분이 있어 전반적인 유용성이 낮습니다.",3521,304,4867, +EV-11,evaluation/evaluation_ai_results_hybrid_v1.csv,1,4.0,3.0,4.0,3.0,2.0,4.0,4.0,4.0,4.0,4.0,2,4,4,2,2,3,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원 동기와 포부는 잘 드러나지만, 구체적인 실행 방법과 JD의 핵심 경험 요구사항이 누락되어 있습니다.",3439,299,6653, +EV-12,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,2.0,3.0,3.0,2.0,3.0,4.0,3.0,4.0,4.0,2,4,4,2,2,2,"[""CONTEXT_IGNORED"",""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","분석에서 중요한 경험 키워드가 누락되었고, 문맥을 무시한 평가가 있었습니다. 전반적으로 개선이 필요합니다.",3806,295,3174, +EV-13,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,3.0,3.0,3.0,2.0,4.0,4.0,4.0,4.0,4.0,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 경험이 직무와 관련된 부분이 부족하며, 주요 키워드가 누락되었습니다. 분석이 부족하여 중요한 문제를 놓쳤습니다.",3511,270,3689, +EV-14,evaluation/evaluation_ai_results_hybrid_v1.csv,0,,,,,,,,,,,2,3,3,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 경험이 JD의 주요 요구사항을 충분히 반영하지 못하고 있으며, 분석이 전혀 이루어지지 않아 중요한 첨삭 대상을 놓쳤습니다.",3764,147,1857, +EV-15,evaluation/evaluation_ai_results_hybrid_v1.csv,0,,,,,,,,,,,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 경험과 강점이 잘 드러나지 않았고, 주요 요구사항이 누락되었습니다. 분석이 전혀 없어서 중요한 문제를 놓쳤습니다.",2927,145,2015, +EV-16,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,3.0,3.0,4.0,2.0,4.0,4.0,4.0,4.0,4.0,2,2,2,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","분석이 부족하여 중요한 문제를 놓쳤고, JD의 핵심 경험 요구사항이 누락되었습니다. 전반적으로 개선이 필요합니다.",3929,293,4032, +EV-17,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,2.0,3.0,3.0,2.0,3.0,4.0,3.0,3.0,4.0,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]",분석이 부족하고 주요 경험 키워드가 누락되어 전반적인 유용성이 낮습니다.,3064,270,3348, +EV-18,evaluation/evaluation_ai_results_hybrid_v1.csv,1,3.0,2.0,4.0,4.0,2.0,4.0,4.0,4.0,4.0,4.0,2,3,3,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","분석에서 병원 행정업무 지원 관련 경험이 부족하다는 점이 명확히 드러났으나, 이에 대한 구체적인 개선이 제시되지 않아 전반적인 유용성이 낮게 평가되었습니다.",3142,281,4608, +EV-19,evaluation/evaluation_ai_results_hybrid_v1.csv,0,,,,,,,,,,,2,4,4,2,2,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 경험과 동기는 명확하나, JD의 주요 요구사항을 충분히 반영하지 못하고 있으며, 분석이 전혀 이루어지지 않아 중요한 첨삭 대상을 놓쳤습니다.",3079,154,2241, +EV-20,evaluation/evaluation_ai_results_hybrid_v1.csv,1,,,,,,,,,,,2,2,2,3,3,2,"[""MISSED_ANALYSIS"",""MISSED_MISSING_KEYWORD""]","지원자의 포부는 있으나 구체적인 실행 방법이 부족하며, JD의 주요 경험 요구사항이 누락되었습니다. 전반적으로 개선이 필요합니다.",3563,183,3287, diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunner.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunner.java index 617bd798..08e531d2 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunner.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunner.java @@ -48,6 +48,9 @@ public class EvaluationAnalysisRunner implements ApplicationRunner { @Value("${evaluation.nlg-judge.enabled:false}") private boolean nlgJudgeEnabled; + @Value("${evaluation.hybrid-merge.enabled:false}") + private boolean hybridMergeEnabled; + private final EvaluationAnalysisBatchService evaluationAnalysisBatchService; private final EvaluationExitCoordinator evaluationExitCoordinator; private final Environment environment; @@ -88,9 +91,9 @@ void validateProfiles() { } void validateExecutionProperties() { - if (nlgJudgeEnabled) { + if (nlgJudgeEnabled || hybridMergeEnabled) { throw new IllegalArgumentException( - "evaluation.analysis.enabled와 evaluation.nlg-judge.enabled를 동시에 true로 설정할 수 없습니다." + "evaluation.analysis.enabled는 evaluation.nlg-judge.enabled 또는 evaluation.hybrid-merge.enabled와 동시에 true로 설정할 수 없습니다." ); } if (!StringUtils.hasText(inputPath)) { diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java index bee1f050..f78c8a89 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java @@ -167,6 +167,24 @@ static void write(Path path, List results) throws IOEx } } + static void writeRows(Path path, List headers, List> rows) throws IOException { + Path parent = path.toAbsolutePath().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + + try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { + writeRow(writer, headers == null ? List.of() : headers); + for (Map row : rows == null ? List.>of() : rows) { + List values = new ArrayList<>(); + for (String header : headers == null ? List.of() : headers) { + values.add(value(row.get(header))); + } + writeRow(writer, values); + } + } + } + private static List> parseRows(String content) { List> rows = new ArrayList<>(); List row = new ArrayList<>(); diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationRunnerFlagValidator.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationRunnerFlagValidator.java index 7a931c45..f561505c 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationRunnerFlagValidator.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationRunnerFlagValidator.java @@ -14,11 +14,18 @@ class EvaluationRunnerFlagValidator implements SmartInitializingSingleton { @Value("${evaluation.nlg-judge.enabled:false}") private boolean nlgJudgeEnabled; + @Value("${evaluation.hybrid-merge.enabled:false}") + private boolean hybridMergeEnabled; + @Override public void afterSingletonsInstantiated() { - if (analysisEvaluationEnabled && nlgJudgeEnabled) { + int enabledRunnerCount = 0; + enabledRunnerCount += analysisEvaluationEnabled ? 1 : 0; + enabledRunnerCount += nlgJudgeEnabled ? 1 : 0; + enabledRunnerCount += hybridMergeEnabled ? 1 : 0; + if (enabledRunnerCount > 1) { throw new IllegalStateException( - "evaluation.analysis.enabled와 evaluation.nlg-judge.enabled를 동시에 true로 설정할 수 없습니다." + "analysis-eval runner flags are mutually exclusive: evaluation.analysis.enabled, evaluation.nlg-judge.enabled, evaluation.hybrid-merge.enabled" ); } } diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeRunner.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeRunner.java new file mode 100644 index 00000000..ecfd665c --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeRunner.java @@ -0,0 +1,136 @@ +package com.jobdri.jobdri_api.domain.analysis.evaluation; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Profile; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; + +@Component +@Profile("analysis-eval") +@ConditionalOnProperty( + prefix = "evaluation.hybrid-merge", + name = "enabled", + havingValue = "true", + matchIfMissing = false +) +@RequiredArgsConstructor +@Slf4j +public class HybridExactMergeRunner implements ApplicationRunner { + + @Value("${evaluation.hybrid-merge.single-pass-input:}") + private String singlePassInputPath; + + @Value("${evaluation.hybrid-merge.two-pass-input:}") + private String twoPassInputPath; + + @Value("${evaluation.hybrid-merge.output:}") + private String outputPath; + + @Value("${evaluation.analysis.enabled:false}") + private boolean analysisEvaluationEnabled; + + @Value("${evaluation.nlg-judge.enabled:false}") + private boolean nlgJudgeEnabled; + + private final HybridExactMergeService hybridExactMergeService; + private final EvaluationExitCoordinator evaluationExitCoordinator; + private final Environment environment; + + @Override + public void run(ApplicationArguments args) throws Exception { + log.info("HybridExactMergeRunner run entered."); + try { + validateProfiles(); + validateExecutionProperties(); + log.info( + "Hybrid exact offline merge를 시작합니다. singlePassInput={}, twoPassInput={}, output={}", + singlePassInputPath, + twoPassInputPath, + outputPath + ); + HybridExactMergeService.HybridExactMergeSummary summary = hybridExactMergeService.merge( + Path.of(singlePassInputPath), + Path.of(twoPassInputPath), + Path.of(outputPath) + ); + validateOutputFile(summary.outputPath(), summary.mergedCases()); + log.info( + "Hybrid exact offline merge 완료. singlePassCases={}, twoPassCases={}, mergedCases={}, output={}", + summary.singlePassCases(), + summary.twoPassCases(), + summary.mergedCases(), + summary.outputPath() + ); + } catch (Exception e) { + log.error("Hybrid exact offline merge 실행에 실패했습니다. message={}", e.getMessage(), e); + evaluationExitCoordinator.exit("hybrid-exact-merge", 1); + throw e; + } + evaluationExitCoordinator.exit("hybrid-exact-merge", 0); + } + + void validateProfiles() { + List profiles = Arrays.asList(environment.getActiveProfiles()); + if (profiles.contains("prod")) { + throw new IllegalStateException("Hybrid exact merge must not run with prod profile."); + } + if (!profiles.contains("analysis-eval")) { + throw new IllegalStateException("Hybrid exact merge requires analysis-eval profile."); + } + } + + void validateExecutionProperties() { + if (analysisEvaluationEnabled || nlgJudgeEnabled) { + throw new IllegalArgumentException( + "evaluation.hybrid-merge.enabled는 evaluation.analysis.enabled 또는 evaluation.nlg-judge.enabled와 동시에 true로 설정할 수 없습니다." + ); + } + if (!StringUtils.hasText(singlePassInputPath)) { + throw new IllegalArgumentException("evaluation.hybrid-merge.single-pass-input 값을 지정해야 합니다."); + } + if (!StringUtils.hasText(twoPassInputPath)) { + throw new IllegalArgumentException("evaluation.hybrid-merge.two-pass-input 값을 지정해야 합니다."); + } + if (!StringUtils.hasText(outputPath)) { + throw new IllegalArgumentException("evaluation.hybrid-merge.output 값을 지정해야 합니다."); + } + if (!Files.isRegularFile(Path.of(singlePassInputPath))) { + throw new IllegalArgumentException( + "evaluation.hybrid-merge.single-pass-input 파일을 찾을 수 없습니다. path=" + singlePassInputPath + ); + } + if (!Files.isRegularFile(Path.of(twoPassInputPath))) { + throw new IllegalArgumentException( + "evaluation.hybrid-merge.two-pass-input 파일을 찾을 수 없습니다. path=" + twoPassInputPath + ); + } + } + + private void validateOutputFile(Path output, int expectedRows) throws java.io.IOException { + if (!Files.isRegularFile(output) || Files.size(output) == 0) { + throw new IllegalStateException("Hybrid exact offline merge output CSV가 생성되지 않았거나 비어 있습니다. path=" + output); + } + int actualRows = EvaluationCsvSupport.read(output).size(); + if (actualRows != expectedRows) { + throw new IllegalStateException( + "Hybrid exact offline merge output row count mismatch. expected=" + + expectedRows + + ", actual=" + + actualRows + + ", path=" + + output + ); + } + } +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java new file mode 100644 index 00000000..9848e5de --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java @@ -0,0 +1,171 @@ +package com.jobdri.jobdri_api.domain.analysis.evaluation; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@RequiredArgsConstructor +@Slf4j +class HybridExactMergeService { + private static final List REQUIRED_SINGLE_PASS_HEADERS = List.of( + "caseId", + "aiScore", + "aiJobFit", + "aiImpact", + "aiCompleteness", + "aiFeedback", + "aiMissingKeywordsJson", + "aiQuestionAnalysesJson", + "rawLlmResponseJson", + "errorMessage", + "createdAt" + ); + private static final List REQUIRED_TWO_PASS_HEADERS = List.of( + "caseId", + "aiMissingKeywordsJson", + "rawLlmResponseJson", + "errorMessage" + ); + + private final ObjectMapper objectMapper; + + HybridExactMergeSummary merge(Path singlePassInput, Path twoPassInput, Path output) throws IOException { + validateDifferentFiles(singlePassInput, output, "single-pass input"); + validateDifferentFiles(twoPassInput, output, "two-pass input"); + + List singleHeaders = EvaluationCsvSupport.readHeaders(singlePassInput); + List twoPassHeaders = EvaluationCsvSupport.readHeaders(twoPassInput); + validateHeaders("single-pass", singleHeaders, REQUIRED_SINGLE_PASS_HEADERS); + validateHeaders("two-pass", twoPassHeaders, REQUIRED_TWO_PASS_HEADERS); + + List> singleRows = EvaluationCsvSupport.read(singlePassInput); + List> twoPassRows = EvaluationCsvSupport.read(twoPassInput); + Map> singleByCaseId = indexByCaseId("single-pass", singleRows); + Map> twoPassByCaseId = indexByCaseId("two-pass", twoPassRows); + validateCaseIdSets(singleByCaseId.keySet(), twoPassByCaseId.keySet()); + + List> mergedRows = new ArrayList<>(); + for (Map singleRow : singleRows) { + String caseId = value(singleRow, "caseId"); + Map twoPassRow = twoPassByCaseId.get(caseId); + validateSuccessRow("single-pass", caseId, singleRow); + validateSuccessRow("two-pass", caseId, twoPassRow); + validateJson("single-pass aiQuestionAnalysesJson", caseId, value(singleRow, "aiQuestionAnalysesJson")); + validateJson("single-pass rawLlmResponseJson", caseId, value(singleRow, "rawLlmResponseJson")); + validateJson("single-pass aiMissingKeywordsJson", caseId, value(singleRow, "aiMissingKeywordsJson")); + validateJson("two-pass aiMissingKeywordsJson", caseId, value(twoPassRow, "aiMissingKeywordsJson")); + validateJson("two-pass rawLlmResponseJson", caseId, value(twoPassRow, "rawLlmResponseJson")); + + Map merged = new LinkedHashMap<>(singleRow); + merged.put("aiMissingKeywordsJson", value(twoPassRow, "aiMissingKeywordsJson")); + merged.put("createdAt", createdAt()); + mergedRows.add(merged); + } + + EvaluationCsvSupport.writeRows(output, singleHeaders, mergedRows); + log.info( + "Hybrid exact offline merge completed. singlePassCases={}, twoPassCases={}, mergedCases={}, output={}", + singleRows.size(), + twoPassRows.size(), + mergedRows.size(), + output + ); + return new HybridExactMergeSummary(singleRows.size(), twoPassRows.size(), mergedRows.size(), output); + } + + private void validateHeaders(String source, List headers, List requiredHeaders) { + Set headerSet = new HashSet<>(headers); + List missing = requiredHeaders.stream() + .filter(header -> !headerSet.contains(header)) + .toList(); + if (!missing.isEmpty()) { + throw new IllegalArgumentException(source + " CSV missing required headers: " + missing); + } + } + + private Map> indexByCaseId(String source, List> rows) { + Map> indexed = new LinkedHashMap<>(); + for (Map row : rows) { + String caseId = value(row, "caseId"); + if (!StringUtils.hasText(caseId)) { + throw new IllegalArgumentException(source + " CSV has blank caseId."); + } + if (indexed.putIfAbsent(caseId, row) != null) { + throw new IllegalArgumentException(source + " CSV has duplicate caseId: " + caseId); + } + } + return indexed; + } + + private void validateCaseIdSets(Set singleCaseIds, Set twoPassCaseIds) { + List missingInTwoPass = singleCaseIds.stream() + .filter(caseId -> !twoPassCaseIds.contains(caseId)) + .toList(); + List missingInSinglePass = twoPassCaseIds.stream() + .filter(caseId -> !singleCaseIds.contains(caseId)) + .toList(); + if (!missingInTwoPass.isEmpty() || !missingInSinglePass.isEmpty()) { + throw new IllegalArgumentException( + "Hybrid exact caseId mismatch. missingInTwoPass=" + + missingInTwoPass + + ", missingInSinglePass=" + + missingInSinglePass + ); + } + } + + private void validateSuccessRow(String source, String caseId, Map row) { + String errorMessage = value(row, "errorMessage"); + if (StringUtils.hasText(errorMessage)) { + throw new IllegalArgumentException(source + " row has errorMessage. caseId=" + caseId); + } + } + + private void validateJson(String field, String caseId, String json) { + if (!StringUtils.hasText(json)) { + throw new IllegalArgumentException(field + " is blank. caseId=" + caseId); + } + try { + objectMapper.readTree(json); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException(field + " is not valid JSON. caseId=" + caseId, e); + } + } + + private void validateDifferentFiles(Path input, Path output, String inputName) { + if (input.toAbsolutePath().normalize().equals(output.toAbsolutePath().normalize())) { + throw new IllegalArgumentException("Hybrid exact output must not overwrite " + inputName + "."); + } + } + + private String value(Map row, String key) { + return row == null || row.get(key) == null ? "" : row.get(key).trim(); + } + + private String createdAt() { + return LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + } + + record HybridExactMergeSummary( + int singlePassCases, + int twoPassCases, + int mergedCases, + Path outputPath + ) { + } +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationRunner.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationRunner.java index 3291ca92..80ddea40 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationRunner.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationRunner.java @@ -51,6 +51,9 @@ public class NlgEvaluationRunner implements ApplicationRunner { @Value("${evaluation.analysis.enabled:false}") private boolean analysisEvaluationEnabled; + @Value("${evaluation.hybrid-merge.enabled:false}") + private boolean hybridMergeEnabled; + private final NlgEvaluationBatchService nlgEvaluationBatchService; private final EvaluationExitCoordinator evaluationExitCoordinator; private final Environment environment; @@ -170,9 +173,9 @@ private void validateCommonOutput() { } private void validateMutuallyExclusiveRunner() { - if (analysisEvaluationEnabled) { + if (analysisEvaluationEnabled || hybridMergeEnabled) { throw new IllegalArgumentException( - "evaluation.analysis.enabled와 evaluation.nlg-judge.enabled를 동시에 true로 설정할 수 없습니다." + "evaluation.nlg-judge.enabled는 evaluation.analysis.enabled 또는 evaluation.hybrid-merge.enabled와 동시에 true로 설정할 수 없습니다." ); } } diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java index 214603d0..87ed765c 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java @@ -25,6 +25,7 @@ import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.StructuredResponse; import com.openai.models.responses.StructuredResponseOutputMessage; +import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -288,6 +289,14 @@ public class AnalysisAiClient { @Value("${analysis.two-pass.enabled:false}") private boolean twoPassEnabled; + @Value("${analysis.mode:}") + private String analysisMode; + + @PostConstruct + void validateAnalysisModeProperty() { + resolveAnalysisMode(); + } + public AnalysisLlmResponse analyze(AnalysisExecutionPayload payload) { return analyze(payload.jobPosting(), payload.answeredQuestions(), payload.jobCategoryEvaluationCriteria()); } @@ -309,20 +318,27 @@ public AnalysisLlmResponse analyze( log.debug("analysis retrieval exception", e); } try { - if (twoPassEnabled) { - return analyzeTwoPass( - AnalysisPromptInput.from(jobPosting, questions), + AnalysisPromptInput promptInput = AnalysisPromptInput.from(jobPosting, questions); + return switch (resolveAnalysisMode()) { + case TWO_PASS -> analyzeTwoPass( + promptInput, referenceContext, jobCategoryEvaluationCriteria, "cover-letter-analysis" ).response(); - } - return analyzeSinglePass( - AnalysisPromptInput.from(jobPosting, questions), - referenceContext, - jobCategoryEvaluationCriteria, - "cover-letter-analysis" - ).response(); + case HYBRID_EXACT -> analyzeHybridExact( + promptInput, + referenceContext, + jobCategoryEvaluationCriteria, + "cover-letter-analysis" + ).response(); + case SINGLE_PASS -> analyzeSinglePass( + promptInput, + referenceContext, + jobCategoryEvaluationCriteria, + "cover-letter-analysis" + ).response(); + }; } catch (GeneralException e) { throw e; } catch (Exception e) { @@ -346,20 +362,26 @@ public AnalysisAiCallResult analyzeForEvaluationResult( JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria ) { try { - if (twoPassEnabled) { - return analyzeTwoPass( + return switch (resolveAnalysisMode()) { + case TWO_PASS -> analyzeTwoPass( promptInput, emptyContext(), jobCategoryEvaluationCriteria, "cover-letter-analysis-evaluation" ); - } - return analyzeSinglePass( - promptInput, - emptyContext(), - jobCategoryEvaluationCriteria, - "cover-letter-analysis-evaluation" - ); + case HYBRID_EXACT -> analyzeHybridExact( + promptInput, + emptyContext(), + jobCategoryEvaluationCriteria, + "cover-letter-analysis-evaluation" + ); + case SINGLE_PASS -> analyzeSinglePass( + promptInput, + emptyContext(), + jobCategoryEvaluationCriteria, + "cover-letter-analysis-evaluation" + ); + }; } catch (GeneralException e) { throw e; } catch (Exception e) { @@ -464,6 +486,47 @@ private AnalysisAiCallResult analyzeTwoPass( ); } + private AnalysisAiCallResult analyzeHybridExact( + AnalysisPromptInput promptInput, + RetrievalContext referenceContext, + JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria, + String operationName + ) { + AnalysisAiCallResult singlePassResult = analyzeSinglePass( + promptInput, + referenceContext, + jobCategoryEvaluationCriteria, + operationName + "-single-pass" + ); + AnalysisAiCallResult twoPassResult = analyzeTwoPass( + promptInput, + referenceContext, + jobCategoryEvaluationCriteria, + operationName + "-two-pass" + ); + AnalysisLlmResponse merged = mergeHybridExact( + singlePassResult.response(), + twoPassResult.response() + ); + log.debug( + "Hybrid exact response merged. questionAnalysesSource=single-pass, missingKeywordsSource=two-pass, scoreSource=single-pass, singlePassQuestionAnalyses={}, twoPassQuestionAnalyses={}, mergedQuestionAnalyses={}, singlePassMissingKeywords={}, twoPassMissingKeywords={}, mergedMissingKeywords={}", + size(singlePassResult.response() == null ? null : singlePassResult.response().questionAnalyses()), + size(twoPassResult.response() == null ? null : twoPassResult.response().questionAnalyses()), + size(merged == null ? null : merged.questionAnalyses()), + size(singlePassResult.response() == null ? null : singlePassResult.response().missingKeywords()), + size(twoPassResult.response() == null ? null : twoPassResult.response().missingKeywords()), + size(merged == null ? null : merged.missingKeywords()) + ); + return AnalysisAiCallResult.hybridExact( + merged, + twoPassResult.rawCandidateResponse(), + twoPassResult.sanitizedCandidateResponse(), + twoPassResult.candidateReviewResponse(), + twoPassResult.candidateCallLatencyMs(), + singlePassResult.finalCallLatencyMs() + twoPassResult.finalCallLatencyMs() + ); + } + private T createStructuredResponse(String operationName, String prompt, Class responseType) { var params = ResponseCreateParams.builder() .model(analysisModel) @@ -1782,6 +1845,29 @@ private List buildFinalMissingKeywords( return result; } + AnalysisLlmResponse mergeHybridExact( + AnalysisLlmResponse singlePassResponse, + AnalysisLlmResponse twoPassResponse + ) { + if (singlePassResponse == null) { + return null; + } + return new AnalysisLlmResponse( + singlePassResponse.jobFit(), + singlePassResponse.impact(), + singlePassResponse.completeness(), + singlePassResponse.feedback(), + singlePassResponse.keyStrengths() == null ? List.of() : List.copyOf(singlePassResponse.keyStrengths()), + singlePassResponse.keyWeaknesses() == null ? List.of() : List.copyOf(singlePassResponse.keyWeaknesses()), + twoPassResponse == null || twoPassResponse.missingKeywords() == null + ? List.of() + : List.copyOf(twoPassResponse.missingKeywords()), + singlePassResponse.questionAnalyses() == null + ? List.of() + : List.copyOf(singlePassResponse.questionAnalyses()) + ); + } + private int acceptedDecisionCount(CandidateReviewResponse reviewResponse) { if (reviewResponse == null || reviewResponse.decisions() == null) { return 0; @@ -2078,6 +2164,24 @@ private RetrievalContext emptyContext() { return new RetrievalContext(List.of(), List.of()); } + AnalysisMode resolveAnalysisMode() { + if (StringUtils.hasText(analysisMode)) { + String normalized = analysisMode.trim().replace('-', '_').toUpperCase(java.util.Locale.ROOT); + try { + return AnalysisMode.valueOf(normalized); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Unsupported analysis mode: " + analysisMode); + } + } + return twoPassEnabled ? AnalysisMode.TWO_PASS : AnalysisMode.SINGLE_PASS; + } + + enum AnalysisMode { + SINGLE_PASS, + TWO_PASS, + HYBRID_EXACT + } + private enum RecheckValidationFailureReason { UNKNOWN_CANDIDATE, LOW_PROBLEM_CLARITY, @@ -2124,5 +2228,24 @@ static AnalysisAiCallResult twoPass( finalCallLatencyMs ); } + + static AnalysisAiCallResult hybridExact( + AnalysisLlmResponse response, + AnalysisCandidateResponse rawCandidateResponse, + AnalysisCandidateResponse sanitizedCandidateResponse, + CandidateReviewResponse candidateReviewResponse, + long candidateCallLatencyMs, + long finalCallLatencyMs + ) { + return new AnalysisAiCallResult( + response, + rawCandidateResponse, + sanitizedCandidateResponse, + candidateReviewResponse, + true, + candidateCallLatencyMs, + finalCallLatencyMs + ); + } } } diff --git a/src/main/resources/application-analysis-eval.yaml b/src/main/resources/application-analysis-eval.yaml index 53be5f7e..7b96cca6 100644 --- a/src/main/resources/application-analysis-eval.yaml +++ b/src/main/resources/application-analysis-eval.yaml @@ -74,6 +74,11 @@ evaluation: input: "" output: "" model: ${OPENAI_NLG_JUDGE_MODEL:gpt-4o-mini} + hybrid-merge: + enabled: false + single-pass-input: "" + two-pass-input: "" + output: "" jwt: secret: diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java index 1bc8e3b0..65d8eb7b 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java @@ -139,6 +139,7 @@ void nlgJudgeRunnerIsPublicScannableComponent() { .run(context -> { assertThat(context).doesNotHaveBean(EvaluationAnalysisRunner.class); assertThat(context).hasSingleBean(NlgEvaluationRunner.class); + assertThat(context).doesNotHaveBean(HybridExactMergeRunner.class); assertThat(context.getBeanNamesForType(ApplicationRunner.class)) .containsExactly("nlgEvaluationRunner"); }); @@ -156,6 +157,7 @@ void analysisRunnerIsNotCreatedWhenAnalysisFlagIsFalse() { .run(context -> { assertThat(context).doesNotHaveBean(EvaluationAnalysisRunner.class); assertThat(context).doesNotHaveBean(NlgEvaluationRunner.class); + assertThat(context).doesNotHaveBean(HybridExactMergeRunner.class); assertThat(context.getBeanNamesForType(ApplicationRunner.class)).isEmpty(); }); } @@ -172,6 +174,7 @@ void analysisRunnerIsCreatedWhenAnalysisFlagIsTrue() { .run(context -> { assertThat(context).hasSingleBean(EvaluationAnalysisRunner.class); assertThat(context).doesNotHaveBean(NlgEvaluationRunner.class); + assertThat(context).doesNotHaveBean(HybridExactMergeRunner.class); assertThat(context.getBeanNamesForType(ApplicationRunner.class)) .containsExactly("evaluationAnalysisRunner"); }); @@ -189,6 +192,7 @@ void nlgRunnerIsCreatedWhenNlgJudgeFlagIsTrue() { .run(context -> { assertThat(context).doesNotHaveBean(EvaluationAnalysisRunner.class); assertThat(context).hasSingleBean(NlgEvaluationRunner.class); + assertThat(context).doesNotHaveBean(HybridExactMergeRunner.class); assertThat(context.getBeanNamesForType(ApplicationRunner.class)) .containsExactly("nlgEvaluationRunner"); verifyNoInteractions(context.getBean(EvaluationExitCoordinator.class)); @@ -207,10 +211,30 @@ void noRunnerIsCreatedWhenBothFlagsAreFalse() { .run(context -> { assertThat(context).doesNotHaveBean(EvaluationAnalysisRunner.class); assertThat(context).doesNotHaveBean(NlgEvaluationRunner.class); + assertThat(context).doesNotHaveBean(HybridExactMergeRunner.class); assertThat(context.getBeanNamesForType(ApplicationRunner.class)).isEmpty(); }); } + @Test + @DisplayName("analysis-eval + hybrid-merge.enabled=true이면 Hybrid merge Runner만 생성된다") + void hybridMergeRunnerIsCreatedWhenHybridMergeFlagIsTrue() { + scannedRunnerContext() + .withPropertyValues( + "spring.profiles.active=analysis-eval", + "evaluation.analysis.enabled=false", + "evaluation.nlg-judge.enabled=false", + "evaluation.hybrid-merge.enabled=true" + ) + .run(context -> { + assertThat(context).doesNotHaveBean(EvaluationAnalysisRunner.class); + assertThat(context).doesNotHaveBean(NlgEvaluationRunner.class); + assertThat(context).hasSingleBean(HybridExactMergeRunner.class); + assertThat(context.getBeanNamesForType(ApplicationRunner.class)) + .containsExactly("hybridExactMergeRunner"); + }); + } + @Test @DisplayName("두 플래그가 모두 true이면 설정 오류로 fail-fast 한다") void bothFlagsTrueFailsFast() { @@ -218,11 +242,27 @@ void bothFlagsTrueFailsFast() { .withPropertyValues( "spring.profiles.active=analysis-eval", "evaluation.analysis.enabled=true", - "evaluation.nlg-judge.enabled=true" + "evaluation.nlg-judge.enabled=true", + "evaluation.hybrid-merge.enabled=false" + ) + .run(context -> assertThat(context.getStartupFailure()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("mutually exclusive")); + } + + @Test + @DisplayName("hybrid-merge와 다른 평가 Runner가 동시에 true이면 설정 오류로 fail-fast 한다") + void hybridMergeAndOtherRunnerFlagsTrueFailsFast() { + runnerContext() + .withPropertyValues( + "spring.profiles.active=analysis-eval", + "evaluation.analysis.enabled=false", + "evaluation.nlg-judge.enabled=true", + "evaluation.hybrid-merge.enabled=true" ) .run(context -> assertThat(context.getStartupFailure()) .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("동시에 true")); + .hasMessageContaining("mutually exclusive")); } @Test @@ -453,6 +493,7 @@ void noRunnerIsCreatedOutsideAnalysisEvalProfile() { .run(context -> { assertThat(context).doesNotHaveBean(EvaluationAnalysisRunner.class); assertThat(context).doesNotHaveBean(NlgEvaluationRunner.class); + assertThat(context).doesNotHaveBean(HybridExactMergeRunner.class); assertThat(context.getBeanNamesForType(ApplicationRunner.class)).isEmpty(); }); } @@ -486,6 +527,7 @@ void analysisEvalYamlDisablesDatabaseSideEffects() { assertThat(properties.getProperty("app.corpus.embedding.sync-on-startup")).isEqualTo("false"); assertThat(properties.getProperty("evaluation.analysis.enabled")).isEqualTo("false"); assertThat(properties.getProperty("evaluation.nlg-judge.enabled")).isEqualTo("false"); + assertThat(properties.getProperty("evaluation.hybrid-merge.enabled")).isEqualTo("false"); assertThat(properties.getProperty("payment.toss.client-key")).contains("dummy-evaluation-client-key"); } @@ -581,7 +623,12 @@ private ApplicationContextRunner scannedRunnerContext() { } @Configuration - @Import({EvaluationAnalysisRunner.class, NlgEvaluationRunner.class, EvaluationRunnerFlagValidator.class}) + @Import({ + EvaluationAnalysisRunner.class, + NlgEvaluationRunner.class, + HybridExactMergeRunner.class, + EvaluationRunnerFlagValidator.class + }) static class RunnerConditionTestConfig { @Bean EvaluationAnalysisBatchService evaluationAnalysisBatchService() { @@ -593,6 +640,11 @@ NlgEvaluationBatchService nlgEvaluationBatchService() { return mock(NlgEvaluationBatchService.class); } + @Bean + HybridExactMergeService hybridExactMergeService() { + return mock(HybridExactMergeService.class); + } + @Bean EvaluationExitCoordinator evaluationExitCoordinator() { return mock(EvaluationExitCoordinator.class); @@ -608,6 +660,7 @@ EvaluationExitCoordinator evaluationExitCoordinator() { classes = { EvaluationAnalysisRunner.class, NlgEvaluationRunner.class, + HybridExactMergeRunner.class, EvaluationRunnerFlagValidator.class } ) @@ -623,6 +676,11 @@ NlgEvaluationBatchService nlgEvaluationBatchService() { return mock(NlgEvaluationBatchService.class); } + @Bean + HybridExactMergeService hybridExactMergeService() { + return mock(HybridExactMergeService.class); + } + @Bean EvaluationExitCoordinator evaluationExitCoordinator() { return mock(EvaluationExitCoordinator.class); diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java new file mode 100644 index 00000000..5769a469 --- /dev/null +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java @@ -0,0 +1,203 @@ +package com.jobdri.jobdri_api.domain.analysis.evaluation; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HybridExactMergeServiceTest { + + @TempDir + Path tempDir; + + private final HybridExactMergeService service = new HybridExactMergeService(new ObjectMapper()); + + @Test + @DisplayName("caseId 기준으로 v5-A questionAnalyses와 two-pass missingKeywords를 exact merge한다") + void mergeUsesSinglePassFieldsAndTwoPassMissingKeywords() throws Exception { + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + Path output = tempDir.resolve("hybrid.csv"); + writeSinglePassCsv(single, List.of(row( + "EV-01", + "90", + "[{\"keyword\":\"single\",\"source\":\"mainTask\"}]", + "[{\"questionId\":1,\"sentence\":\"문장, 쉼표\",\"status\":\"MENTIONED\",\"reason\":\"이유\",\"improvement\":null}]", + "{\"jobFit\":90,\"impact\":80,\"completeness\":70,\"feedback\":\"single\",\"keyStrengths\":[{\"title\":\"강점\",\"quote\":\"문장, 쉼표\"}],\"keyWeaknesses\":[],\"missingKeywords\":[],\"questionAnalyses\":[]}", + "" + ))); + writeTwoPassCsv(twoPass, List.of(row( + "EV-01", + "10", + "[{\"keyword\":\"장애 대응 경험\",\"source\":\"qualification\"}]", + "[]", + "{\"jobFit\":10,\"impact\":20,\"completeness\":30,\"feedback\":\"two-pass\",\"keyStrengths\":[],\"keyWeaknesses\":[],\"missingKeywords\":[{\"keyword\":\"장애 대응 경험\",\"source\":\"qualification\"}],\"questionAnalyses\":[]}", + "" + ))); + + HybridExactMergeService.HybridExactMergeSummary summary = service.merge(single, twoPass, output); + + assertThat(summary.singlePassCases()).isEqualTo(1); + assertThat(summary.twoPassCases()).isEqualTo(1); + assertThat(summary.mergedCases()).isEqualTo(1); + List> rows = EvaluationCsvSupport.read(output); + assertThat(rows).hasSize(1); + Map merged = rows.getFirst(); + assertThat(merged.get("aiScore")).isEqualTo("90"); + assertThat(merged.get("aiFeedback")).isEqualTo("single feedback"); + assertThat(merged.get("aiQuestionAnalysesJson")).contains("문장, 쉼표"); + assertThat(merged.get("aiMissingKeywordsJson")).contains("장애 대응 경험"); + assertThat(merged.get("aiMissingKeywordsJson")).doesNotContain("single"); + } + + @Test + @DisplayName("UTF-8 BOM과 CSV quote/newline을 포함한 입력도 병합한다") + void mergeSupportsBomAndQuotedFields() throws Exception { + Path single = tempDir.resolve("single-bom.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + Path output = tempDir.resolve("hybrid.csv"); + Files.writeString( + single, + "\uFEFF" + singleHeader() + "\n" + + csv("EV-01") + "," + + csv("AI·개발·데이터") + "," + + csv("백엔드") + "," + + csv("70") + "," + + csv("70") + "," + + csv("60") + "," + + csv("80") + "," + + csv("single\nfeedback") + "," + + csv("[]") + "," + + csv("[]") + "," + + csv("{\"jobFit\":70,\"impact\":60,\"completeness\":80,\"feedback\":\"single\\nfeedback\",\"keyStrengths\":[],\"keyWeaknesses\":[],\"missingKeywords\":[],\"questionAnalyses\":[]}") + "," + + csv("") + "," + + csv("2026-07-26T10:00:00") + "\n", + StandardCharsets.UTF_8 + ); + writeTwoPassCsv(twoPass, List.of(row( + "EV-01", + "70", + "[{\"keyword\":\"장애 대응\\n경험\",\"source\":\"qualification\"}]", + "[]", + "{\"jobFit\":70,\"impact\":60,\"completeness\":80,\"feedback\":\"two-pass\",\"keyStrengths\":[],\"keyWeaknesses\":[],\"missingKeywords\":[],\"questionAnalyses\":[]}", + "" + ))); + + service.merge(single, twoPass, output); + + Map merged = EvaluationCsvSupport.read(output).getFirst(); + assertThat(merged.get("caseId")).isEqualTo("EV-01"); + assertThat(merged.get("aiFeedback")).isEqualTo("single\nfeedback"); + assertThat(merged.get("aiMissingKeywordsJson")).contains("장애 대응\\n경험"); + } + + @Test + @DisplayName("single-pass 또는 two-pass에 caseId가 누락되면 실패한다") + void mergeFailsOnCaseIdMismatch() throws Exception { + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + Path output = tempDir.resolve("hybrid.csv"); + writeSinglePassCsv(single, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + writeTwoPassCsv(twoPass, List.of(row("EV-02", "90", "[]", "[]", validRawJson(), ""))); + + assertThatThrownBy(() -> service.merge(single, twoPass, output)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("caseId mismatch"); + } + + @Test + @DisplayName("중복 caseId가 있으면 실패한다") + void mergeFailsOnDuplicateCaseId() throws Exception { + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + Path output = tempDir.resolve("hybrid.csv"); + writeSinglePassCsv(single, List.of( + row("EV-01", "90", "[]", "[]", validRawJson(), ""), + row("EV-01", "80", "[]", "[]", validRawJson(), "") + )); + writeTwoPassCsv(twoPass, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + + assertThatThrownBy(() -> service.merge(single, twoPass, output)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("duplicate caseId"); + } + + @Test + @DisplayName("필수 JSON 필드가 파싱되지 않으면 실패한다") + void mergeFailsOnInvalidJson() throws Exception { + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + Path output = tempDir.resolve("hybrid.csv"); + writeSinglePassCsv(single, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + writeTwoPassCsv(twoPass, List.of(row("EV-01", "90", "not-json", "[]", validRawJson(), ""))); + + assertThatThrownBy(() -> service.merge(single, twoPass, output)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("two-pass aiMissingKeywordsJson is not valid JSON"); + } + + private void writeSinglePassCsv(Path path, List> rows) throws Exception { + EvaluationCsvSupport.writeRows(path, List.of(singleHeader().split(",")), rows); + } + + private void writeTwoPassCsv(Path path, List> rows) throws Exception { + EvaluationCsvSupport.writeRows(path, List.of(twoPassHeader().split(",")), rows); + } + + private String singleHeader() { + return "caseId,jobCategoryMiddle,jobCategorySmall,aiScore,aiJobFit,aiImpact,aiCompleteness,aiFeedback,aiMissingKeywordsJson,aiQuestionAnalysesJson,rawLlmResponseJson,errorMessage,createdAt"; + } + + private String twoPassHeader() { + return "caseId,jobCategoryMiddle,jobCategorySmall,aiScore,aiJobFit,aiImpact,aiCompleteness,aiFeedback,aiMissingKeywordsJson,aiQuestionAnalysesJson,rawLlmResponseJson,rawCandidateResponseJson,sanitizedCandidateResponseJson,candidateReviewResponseJson,errorMessage,createdAt"; + } + + private Map row( + String caseId, + String score, + String missingKeywordsJson, + String questionAnalysesJson, + String rawLlmResponseJson, + String errorMessage + ) { + Map row = new java.util.LinkedHashMap<>(); + row.put("caseId", caseId); + row.put("jobCategoryMiddle", "AI·개발·데이터"); + row.put("jobCategorySmall", "백엔드"); + row.put("aiScore", score); + row.put("aiJobFit", score); + row.put("aiImpact", "60"); + row.put("aiCompleteness", "70"); + row.put("aiFeedback", "single feedback"); + row.put("aiMissingKeywordsJson", missingKeywordsJson); + row.put("aiQuestionAnalysesJson", questionAnalysesJson); + row.put("rawLlmResponseJson", rawLlmResponseJson); + row.put("rawCandidateResponseJson", "{}"); + row.put("sanitizedCandidateResponseJson", "{}"); + row.put("candidateReviewResponseJson", "{}"); + row.put("errorMessage", errorMessage); + row.put("createdAt", "2026-07-26T10:00:00"); + return row; + } + + private String validRawJson() { + return "{\"jobFit\":70,\"impact\":60,\"completeness\":80,\"feedback\":\"ok\",\"keyStrengths\":[],\"keyWeaknesses\":[],\"missingKeywords\":[],\"questionAnalyses\":[]}"; + } + + private String csv(String value) { + String safeValue = value == null ? "" : value; + if (safeValue.contains(",") || safeValue.contains("\"") || safeValue.contains("\n") || safeValue.contains("\r")) { + return "\"" + safeValue.replace("\"", "\"\"") + "\""; + } + return safeValue; + } +} diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java index 7b3421c4..a6706ac0 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java @@ -18,11 +18,13 @@ import com.openai.client.OpenAIClient; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; import java.util.List; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -37,6 +39,125 @@ class AnalysisAiClientTest { new ObjectMapper() ); + @Test + @DisplayName("analysis.mode가 없으면 기존 two-pass boolean으로 분석 모드를 해석한다") + void resolveAnalysisModeFallsBackToTwoPassBoolean() { + ReflectionTestUtils.setField(analysisAiClient, "analysisMode", ""); + ReflectionTestUtils.setField(analysisAiClient, "twoPassEnabled", false); + assertThat(analysisAiClient.resolveAnalysisMode()).isEqualTo(AnalysisAiClient.AnalysisMode.SINGLE_PASS); + + ReflectionTestUtils.setField(analysisAiClient, "twoPassEnabled", true); + assertThat(analysisAiClient.resolveAnalysisMode()).isEqualTo(AnalysisAiClient.AnalysisMode.TWO_PASS); + } + + @Test + @DisplayName("analysis.mode가 기존 two-pass boolean보다 우선한다") + void analysisModePropertyTakesPrecedenceOverTwoPassBoolean() { + ReflectionTestUtils.setField(analysisAiClient, "twoPassEnabled", true); + ReflectionTestUtils.setField(analysisAiClient, "analysisMode", "single-pass"); + assertThat(analysisAiClient.resolveAnalysisMode()).isEqualTo(AnalysisAiClient.AnalysisMode.SINGLE_PASS); + + ReflectionTestUtils.setField(analysisAiClient, "analysisMode", "hybrid-exact"); + assertThat(analysisAiClient.resolveAnalysisMode()).isEqualTo(AnalysisAiClient.AnalysisMode.HYBRID_EXACT); + } + + @Test + @DisplayName("지원하지 않는 analysis.mode는 명확한 예외를 던진다") + void unsupportedAnalysisModeFailsFast() { + ReflectionTestUtils.setField(analysisAiClient, "analysisMode", "unknown"); + + assertThatThrownBy(analysisAiClient::resolveAnalysisMode) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unsupported analysis mode: unknown"); + } + + @Test + @DisplayName("Hybrid Exact merge는 questionAnalyses와 점수는 single-pass, missingKeywords는 two-pass 전체 결과를 사용한다") + void mergeHybridExactUsesExplicitSources() { + AnalysisLlmResponse.QuestionAnalysisItem singleQuestionAnalysis = + new AnalysisLlmResponse.QuestionAnalysisItem(1L, "Spring Boot API를 개발했습니다.", "MENTIONED", "결과가 부족합니다.", null); + AnalysisLlmResponse.QuestionAnalysisItem twoPassQuestionAnalysis = + new AnalysisLlmResponse.QuestionAnalysisItem(1L, "장애 대응 경험이 있습니다.", "MENTIONED", "역할이 부족합니다.", null); + AnalysisLlmResponse.MissingKeywordItem singleMissingKeyword = + new AnalysisLlmResponse.MissingKeywordItem("single 누락", "mainTask"); + AnalysisLlmResponse.MissingKeywordItem twoPassMissingKeyword = + new AnalysisLlmResponse.MissingKeywordItem("장애 대응 경험", "qualification"); + AnalysisLlmResponse singlePassResponse = new AnalysisLlmResponse( + 80, + 70, + 60, + "single feedback", + List.of(new AnalysisLlmResponse.HighlightItem("강점", "Spring Boot API")), + List.of(new AnalysisLlmResponse.HighlightItem("약점", "결과")), + List.of(singleMissingKeyword), + List.of(singleQuestionAnalysis) + ); + AnalysisLlmResponse twoPassResponse = new AnalysisLlmResponse( + 10, + 20, + 30, + "two-pass feedback", + List.of(new AnalysisLlmResponse.HighlightItem("two-pass 강점", "장애 대응")), + List.of(), + List.of(twoPassMissingKeyword), + List.of(twoPassQuestionAnalysis) + ); + + AnalysisLlmResponse merged = analysisAiClient.mergeHybridExact( + singlePassResponse, + twoPassResponse + ); + + assertThat(merged.jobFit()).isEqualTo(80); + assertThat(merged.impact()).isEqualTo(70); + assertThat(merged.completeness()).isEqualTo(60); + assertThat(merged.feedback()).isEqualTo("single feedback"); + assertThat(merged.keyStrengths()).isEqualTo(singlePassResponse.keyStrengths()); + assertThat(merged.questionAnalyses()).containsExactly(singleQuestionAnalysis); + assertThat(merged.missingKeywords()).containsExactly(twoPassMissingKeyword); + assertThat(merged.missingKeywords()).doesNotContain(singleMissingKeyword); + assertThat(merged.questionAnalyses()).doesNotContain(twoPassQuestionAnalysis); + } + + @Test + @DisplayName("Hybrid Exact merge는 source 응답 리스트를 방어적으로 복사한다") + void mergeHybridExactCopiesLists() { + AnalysisLlmResponse.QuestionAnalysisItem singleQuestionAnalysis = + new AnalysisLlmResponse.QuestionAnalysisItem(1L, "Spring Boot API를 개발했습니다.", "MENTIONED", "결과가 부족합니다.", null); + AnalysisLlmResponse.MissingKeywordItem twoPassMissingKeyword = + new AnalysisLlmResponse.MissingKeywordItem("장애 대응 경험", "qualification"); + AnalysisLlmResponse singlePassResponse = new AnalysisLlmResponse( + 80, + 70, + 60, + "single feedback", + new java.util.ArrayList<>(List.of(new AnalysisLlmResponse.HighlightItem("강점", "Spring Boot API"))), + new java.util.ArrayList<>(), + new java.util.ArrayList<>(), + new java.util.ArrayList<>(List.of(singleQuestionAnalysis)) + ); + AnalysisLlmResponse twoPassResponse = new AnalysisLlmResponse( + 10, + 20, + 30, + "two-pass feedback", + new java.util.ArrayList<>(), + new java.util.ArrayList<>(), + new java.util.ArrayList<>(List.of(twoPassMissingKeyword)), + new java.util.ArrayList<>() + ); + + AnalysisLlmResponse merged = analysisAiClient.mergeHybridExact(singlePassResponse, twoPassResponse); + + assertThat(merged.keyStrengths()).containsExactlyElementsOf(singlePassResponse.keyStrengths()); + assertThat(merged.questionAnalyses()).containsExactly(singleQuestionAnalysis); + assertThat(merged.missingKeywords()).containsExactly(twoPassMissingKeyword); + assertThatThrownBy(() -> merged.questionAnalyses().add(singleQuestionAnalysis)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> merged.missingKeywords().add(twoPassMissingKeyword)) + .isInstanceOf(UnsupportedOperationException.class); + } + @Test @DisplayName("직무 중분류 기준이 있으면 프롬프트에 보조 평가 기준 섹션을 포함한다") void buildPromptIncludesJobCategoryCriteriaWhenPresent() { From ecfe9f4a9f4e76fdbe97c479c1ad881b8ae2b283 Mon Sep 17 00:00:00 2001 From: wooh Date: Sun, 26 Jul 2026 03:24:00 +0900 Subject: [PATCH 2/2] =?UTF-8?q?[Fix]=20Hybrid=20Exact=20=EB=A6=AC=EB=B7=B0?= =?UTF-8?q?=20=EC=9C=A0=ED=9A=A8=20=ED=95=AD=EB=AA=A9=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 평가 CSV writer의 공통 파일 생성 및 writer 처리 흐름 추출 - Hybrid Exact offline merge 입력 경로 동일성 검증 추가 - unsupported analysis.mode 예외 cause 보존 - Hybrid merge Runner 실패 시 종료 요청 및 예외 전파 테스트 추가 - Hybrid merge error row 및 overwrite 방지 테스트 추가 - 병렬화 요구는 Hybrid Exact 기준 구현 제약과 충돌하여 미반영 --- .../evaluation/EvaluationCsvSupport.java | 129 +++++++++--------- .../evaluation/HybridExactMergeService.java | 4 + .../analysis/service/AnalysisAiClient.java | 2 +- .../EvaluationAnalysisRunnerSafetyTest.java | 27 ++++ .../HybridExactMergeServiceTest.java | 54 ++++++++ .../service/AnalysisAiClientTest.java | 4 +- 6 files changed, 156 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java index f78c8a89..c70343bc 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationCsvSupport.java @@ -63,60 +63,53 @@ private static CsvRows readRows(Path path) throws IOException { } static void write(Path path, List results) throws IOException { - Path parent = path.toAbsolutePath().getParent(); - if (parent != null) { - Files.createDirectories(parent); - } - - try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { - writeRow(writer, List.of( - "caseId", - "jobCategoryMiddle", - "jobCategorySmall", - "mainTasks", - "qualifications", - "preferences", - "question", - "answer", - "aiScore", - "aiJobFit", - "aiImpact", - "aiCompleteness", - "aiFeedback", - "aiMissingKeywordsJson", - "aiQuestionAnalysesJson", - "rawLlmResponseJson", - "rawCandidateResponseJson", - "sanitizedCandidateResponseJson", - "candidateReviewResponseJson", - "candidateCount", - "candidateAnalysisCount", - "candidateStrengthCount", - "candidateMissingKeywordCount", - "acceptedCandidateCount", - "rejectedCandidateCount", - "rejectionCodeCounts", - "finalAnalysisCount", - "strengthCandidateCount", - "finalStrengthCount", - "missingKeywordCandidateCount", - "finalMissingKeywordCount", - "candidateCallLatencyMs", - "finalCallLatencyMs", - "candidateLatencyMs", - "finalLatencyMs", - "candidateInputTokens", - "candidateOutputTokens", - "finalInputTokens", - "finalOutputTokens", - "totalInputTokens", - "totalOutputTokens", - "failureStage", - "errorMessage", - "createdAt" - )); - - for (EvaluationAnalysisResult result : results) { + writeCsv(path, List.of( + "caseId", + "jobCategoryMiddle", + "jobCategorySmall", + "mainTasks", + "qualifications", + "preferences", + "question", + "answer", + "aiScore", + "aiJobFit", + "aiImpact", + "aiCompleteness", + "aiFeedback", + "aiMissingKeywordsJson", + "aiQuestionAnalysesJson", + "rawLlmResponseJson", + "rawCandidateResponseJson", + "sanitizedCandidateResponseJson", + "candidateReviewResponseJson", + "candidateCount", + "candidateAnalysisCount", + "candidateStrengthCount", + "candidateMissingKeywordCount", + "acceptedCandidateCount", + "rejectedCandidateCount", + "rejectionCodeCounts", + "finalAnalysisCount", + "strengthCandidateCount", + "finalStrengthCount", + "missingKeywordCandidateCount", + "finalMissingKeywordCount", + "candidateCallLatencyMs", + "finalCallLatencyMs", + "candidateLatencyMs", + "finalLatencyMs", + "candidateInputTokens", + "candidateOutputTokens", + "finalInputTokens", + "finalOutputTokens", + "totalInputTokens", + "totalOutputTokens", + "failureStage", + "errorMessage", + "createdAt" + ), writer -> { + for (EvaluationAnalysisResult result : results == null ? List.of() : results) { writeRow(writer, List.of( value(result.caseId()), value(result.jobCategoryMiddle()), @@ -164,10 +157,23 @@ static void write(Path path, List results) throws IOEx value(result.createdAt()) )); } - } + }); } static void writeRows(Path path, List headers, List> rows) throws IOException { + List safeHeaders = headers == null ? List.of() : headers; + writeCsv(path, safeHeaders, writer -> { + for (Map row : rows == null ? List.>of() : rows) { + List values = new ArrayList<>(); + for (String header : safeHeaders) { + values.add(value(row.get(header))); + } + writeRow(writer, values); + } + }); + } + + private static void writeCsv(Path path, List headers, CsvRowsWriter rowsWriter) throws IOException { Path parent = path.toAbsolutePath().getParent(); if (parent != null) { Files.createDirectories(parent); @@ -175,16 +181,15 @@ static void writeRows(Path path, List headers, List> try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { writeRow(writer, headers == null ? List.of() : headers); - for (Map row : rows == null ? List.>of() : rows) { - List values = new ArrayList<>(); - for (String header : headers == null ? List.of() : headers) { - values.add(value(row.get(header))); - } - writeRow(writer, values); - } + rowsWriter.write(writer); } } + @FunctionalInterface + private interface CsvRowsWriter { + void write(BufferedWriter writer) throws IOException; + } + private static List> parseRows(String content) { List> rows = new ArrayList<>(); List row = new ArrayList<>(); diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java index 9848e5de..68ec70a5 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeService.java @@ -45,6 +45,7 @@ class HybridExactMergeService { private final ObjectMapper objectMapper; HybridExactMergeSummary merge(Path singlePassInput, Path twoPassInput, Path output) throws IOException { + validateDifferentFiles(singlePassInput, twoPassInput, "single-pass input and two-pass input"); validateDifferentFiles(singlePassInput, output, "single-pass input"); validateDifferentFiles(twoPassInput, output, "two-pass input"); @@ -149,6 +150,9 @@ private void validateJson(String field, String caseId, String json) { private void validateDifferentFiles(Path input, Path output, String inputName) { if (input.toAbsolutePath().normalize().equals(output.toAbsolutePath().normalize())) { + if ("single-pass input and two-pass input".equals(inputName)) { + throw new IllegalArgumentException("Hybrid exact input paths must be different: " + inputName + "."); + } throw new IllegalArgumentException("Hybrid exact output must not overwrite " + inputName + "."); } } diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java index 87ed765c..bffd05b1 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClient.java @@ -2170,7 +2170,7 @@ AnalysisMode resolveAnalysisMode() { try { return AnalysisMode.valueOf(normalized); } catch (IllegalArgumentException e) { - throw new IllegalStateException("Unsupported analysis mode: " + analysisMode); + throw new IllegalStateException("Unsupported analysis mode: " + analysisMode, e); } } return twoPassEnabled ? AnalysisMode.TWO_PASS : AnalysisMode.SINGLE_PASS; diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java index 65d8eb7b..8874e67e 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/EvaluationAnalysisRunnerSafetyTest.java @@ -265,6 +265,33 @@ void hybridMergeAndOtherRunnerFlagsTrueFailsFast() { .hasMessageContaining("mutually exclusive")); } + @Test + @DisplayName("Hybrid merge Runner는 실행 실패 시 실패 종료를 요청하고 원래 예외를 전파한다") + void hybridMergeRunRequestsFailureExitAndRethrowsOriginalException() throws Exception { + HybridExactMergeService mergeService = mock(HybridExactMergeService.class); + EvaluationExitCoordinator exitCoordinator = mock(EvaluationExitCoordinator.class); + Environment environment = mock(Environment.class); + when(environment.getActiveProfiles()).thenReturn(new String[]{"analysis-eval"}); + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + Path output = tempDir.resolve("hybrid.csv"); + Files.writeString(single, "caseId\nEV-01\n"); + Files.writeString(twoPass, "caseId\nEV-01\n"); + RuntimeException failure = new RuntimeException("merge failed"); + when(mergeService.merge(single, twoPass, output)).thenThrow(failure); + HybridExactMergeRunner runner = new HybridExactMergeRunner(mergeService, exitCoordinator, environment); + ReflectionTestUtils.setField(runner, "singlePassInputPath", single.toString()); + ReflectionTestUtils.setField(runner, "twoPassInputPath", twoPass.toString()); + ReflectionTestUtils.setField(runner, "outputPath", output.toString()); + ReflectionTestUtils.setField(runner, "analysisEvaluationEnabled", false); + ReflectionTestUtils.setField(runner, "nlgJudgeEnabled", false); + + assertThatThrownBy(() -> runner.run(new DefaultApplicationArguments())) + .isSameAs(failure); + + verify(exitCoordinator).exit("hybrid-exact-merge", 1); + } + @Test @DisplayName("NLG judge 비교 모드는 OpenAI 키와 비용 확인 없이 입력 파일과 출력 경로만 검증한다") void nlgJudgeComparisonModeDoesNotRequireOpenAiProperties() throws Exception { diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java index 5769a469..e824d952 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/HybridExactMergeServiceTest.java @@ -145,6 +145,60 @@ void mergeFailsOnInvalidJson() throws Exception { .hasMessageContaining("two-pass aiMissingKeywordsJson is not valid JSON"); } + @Test + @DisplayName("입력 row에 errorMessage가 있으면 병합하지 않는다") + void mergeFailsOnErrorRow() throws Exception { + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + Path output = tempDir.resolve("hybrid.csv"); + writeSinglePassCsv(single, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), "single failed"))); + writeTwoPassCsv(twoPass, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + + assertThatThrownBy(() -> service.merge(single, twoPass, output)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("single-pass row has errorMessage") + .hasMessageContaining("EV-01"); + } + + @Test + @DisplayName("output이 single-pass 입력과 같으면 거부한다") + void mergeFailsWhenOutputOverwritesSinglePassInput() throws Exception { + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + writeSinglePassCsv(single, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + writeTwoPassCsv(twoPass, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + + assertThatThrownBy(() -> service.merge(single, twoPass, single)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Hybrid exact output must not overwrite single-pass input"); + } + + @Test + @DisplayName("output이 two-pass 입력과 같으면 거부한다") + void mergeFailsWhenOutputOverwritesTwoPassInput() throws Exception { + Path single = tempDir.resolve("single.csv"); + Path twoPass = tempDir.resolve("two-pass.csv"); + writeSinglePassCsv(single, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + writeTwoPassCsv(twoPass, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + + assertThatThrownBy(() -> service.merge(single, twoPass, twoPass)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Hybrid exact output must not overwrite two-pass input"); + } + + @Test + @DisplayName("single-pass 입력과 two-pass 입력이 같으면 거부한다") + void mergeFailsWhenInputFilesAreSame() throws Exception { + Path input = tempDir.resolve("same.csv"); + Path output = tempDir.resolve("hybrid.csv"); + writeSinglePassCsv(input, List.of(row("EV-01", "90", "[]", "[]", validRawJson(), ""))); + + assertThatThrownBy(() -> service.merge(input, input, output)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Hybrid exact input paths must be different") + .hasMessageContaining("single-pass input and two-pass input"); + } + private void writeSinglePassCsv(Path path, List> rows) throws Exception { EvaluationCsvSupport.writeRows(path, List.of(singleHeader().split(",")), rows); } diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java index a6706ac0..13d75e00 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/AnalysisAiClientTest.java @@ -68,7 +68,9 @@ void unsupportedAnalysisModeFailsFast() { assertThatThrownBy(analysisAiClient::resolveAnalysisMode) .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Unsupported analysis mode: unknown"); + .hasMessageContaining("Unsupported analysis mode: unknown") + .cause() + .isInstanceOf(IllegalArgumentException.class); } @Test