diff --git a/src/data/flip_labels/flip_labels_llm-router.json b/src/data/flip_labels/flip_labels_llm-router.json index daf98f7..9450917 100644 --- a/src/data/flip_labels/flip_labels_llm-router.json +++ b/src/data/flip_labels/flip_labels_llm-router.json @@ -1,15 +1,15 @@ [ { "global index": "AIME_112", - "flip": 1 + "flip": 0 }, { "global index": "AIME_58", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_12", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_123", @@ -17,7 +17,7 @@ }, { "global index": "ArcMMLU_16", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_182", @@ -29,27 +29,27 @@ }, { "global index": "ArcMMLU_293", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_349", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_378", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_443", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_496", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_631", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_646", @@ -57,47 +57,47 @@ }, { "global index": "ArcMMLU_659", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_676", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_685", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_689", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_702", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_713", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_98", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_1165", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_1347", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_472", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_733", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_0", @@ -109,7 +109,7 @@ }, { "global index": "ChessInstruct_144", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_42", @@ -125,7 +125,7 @@ }, { "global index": "ChessInstruct_84", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_28", @@ -149,11 +149,11 @@ }, { "global index": "Ethics_commonsense_85", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_90", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_deontology_0", @@ -213,7 +213,7 @@ }, { "global index": "FinQA_149", - "flip": 1 + "flip": 0 }, { "global index": "FinQA_208", @@ -221,87 +221,87 @@ }, { "global index": "FinQA_56", - "flip": 1 + "flip": 0 }, { "global index": "FinQA_60", - "flip": 1 + "flip": 0 }, { "global index": "GSM8K_43", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1002", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1094", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1102", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1113", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_124", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1243", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_30", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_502", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_526", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_591", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_766", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_87", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_915", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_944", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_968", - "flip": 1 + "flip": 0 }, { "global index": "GeoGraphyData_100k_27", - "flip": 1 + "flip": 0 }, { "global index": "GeoGraphyData_100k_42", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_105", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_114", @@ -313,19 +313,19 @@ }, { "global index": "LiveCodeBench_131", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_136", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_181", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_237", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_271", @@ -333,11 +333,11 @@ }, { "global index": "LiveCodeBench_350", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_386", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_405", @@ -349,63 +349,63 @@ }, { "global index": "LiveCodeBench_431", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_437", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_476", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_485", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_49", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_491", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_499", - "flip": 1 + "flip": 0 }, { "global index": "MATH_108", - "flip": 1 + "flip": 0 }, { "global index": "MATH_442", - "flip": 1 + "flip": 0 }, { "global index": "MATH_53", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_2808", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_2912", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_2980", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_2985", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_3188", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_3215", @@ -413,23 +413,23 @@ }, { "global index": "MMLUPro_biology_3225", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_226", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_294", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_378", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_430", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_503", @@ -437,107 +437,107 @@ }, { "global index": "MMLUPro_business_507", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_6", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_784", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_3796", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_3837", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_3974", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_4067", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_4407", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9086", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9110", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9136", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9138", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9149", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9200", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9212", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9239", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9264", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9285", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9289", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9414", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9415", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9430", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9452", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9471", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9475", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5769", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5907", @@ -545,7 +545,7 @@ }, { "global index": "MMLUPro_economics_5931", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5965", @@ -553,11 +553,11 @@ }, { "global index": "MMLUPro_economics_6114", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_6122", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_6135", @@ -569,63 +569,63 @@ }, { "global index": "MMLUPro_economics_6353", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10076", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10125", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10179", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10195", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10199", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10298", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10342", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10395", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10428", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10432", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10473", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10537", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10701", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10823", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10864", @@ -633,11 +633,11 @@ }, { "global index": "MMLUPro_health_4885", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_4973", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5093", @@ -645,7 +645,7 @@ }, { "global index": "MMLUPro_health_5144", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5214", @@ -653,27 +653,27 @@ }, { "global index": "MMLUPro_health_5215", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5261", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5473", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5514", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4486", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4490", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4497", @@ -681,15 +681,15 @@ }, { "global index": "MMLUPro_history_4509", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4517", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4523", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4605", @@ -705,11 +705,11 @@ }, { "global index": "MMLUPro_history_4717", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4749", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4752", @@ -717,35 +717,35 @@ }, { "global index": "MMLUPro_history_4774", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4810", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4833", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4836", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4841", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1007", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1031", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1386", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1462", @@ -761,27 +761,27 @@ }, { "global index": "MMLUPro_law_1818", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_806", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_899", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6429", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6526", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6623", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6848", @@ -789,23 +789,23 @@ }, { "global index": "MMLUPro_math_7101", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7249", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7284", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7451", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7577", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_philosophy_9510", @@ -817,7 +817,7 @@ }, { "global index": "MMLUPro_philosophy_9663", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_philosophy_9672", @@ -829,23 +829,23 @@ }, { "global index": "MMLUPro_physics_7773", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_7887", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_7893", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_8888", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_9017", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2005", @@ -865,23 +865,23 @@ }, { "global index": "MMLUPro_psychology_2420", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2450", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2457", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2524", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_121", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_16", @@ -913,7 +913,7 @@ }, { "global index": "MMLU_management_41", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_management_77", @@ -921,51 +921,51 @@ }, { "global index": "MMLU_management_91", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_management_93", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_158", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_1742", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_202", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_2092", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_2102", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_2851", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_827", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_84", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1005", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1054", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1298", @@ -985,15 +985,15 @@ }, { "global index": "MedMCQA_2010", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2323", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2366", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2581", @@ -1005,7 +1005,7 @@ }, { "global index": "MedMCQA_59", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_643", @@ -1013,35 +1013,35 @@ }, { "global index": "MedMCQA_853", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_126", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_14", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_147", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_152", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_188", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_189", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_240", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_33", @@ -1049,95 +1049,95 @@ }, { "global index": "MusicTheoryBench_337", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_340", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_70", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_131", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_1683", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_2474", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_2820", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_3282", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_4102", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_4128", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_4347", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_4540", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_5022", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_5259", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_533", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_5894", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_6829", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_7678", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_7964", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_8215", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_8598", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_927", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Animals_2545", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Animals_262", @@ -1145,7 +1145,7 @@ }, { "global index": "OpenTDB_Art_1429", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Celebrities_1078", @@ -1153,7 +1153,7 @@ }, { "global index": "OpenTDB_Celebrities_1456", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Celebrities_3577", @@ -1169,7 +1169,7 @@ }, { "global index": "OpenTDB_General Knowledge_1807", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_1957", @@ -1177,15 +1177,15 @@ }, { "global index": "OpenTDB_General Knowledge_2181", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_3404", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_3727", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_4019", @@ -1197,27 +1197,27 @@ }, { "global index": "OpenTDB_Geography_2346", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Geography_3880", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Geography_792", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_1162", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_2026", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_3712", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_3902", @@ -1225,19 +1225,19 @@ }, { "global index": "OpenTDB_Science & Nature_1175", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Science & Nature_1560", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Science & Nature_3716", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Science & Nature_476", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Sports_2289", @@ -1245,19 +1245,19 @@ }, { "global index": "OpenTDB_Vehicles_1173", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Vehicles_1419", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Vehicles_2519", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Vehicles_3234", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_0", @@ -1265,19 +1265,19 @@ }, { "global index": "PubMedQA_154", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_18", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_238", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_250", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_337", @@ -1285,23 +1285,23 @@ }, { "global index": "PubMedQA_362", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_437", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_510", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_520", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_575", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_582", @@ -1309,11 +1309,11 @@ }, { "global index": "PubMedQA_588", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_610", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_63", @@ -1321,35 +1321,35 @@ }, { "global index": "PubMedQA_643", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_687", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_722", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_73", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_755", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_8", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_81", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_854", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_905", @@ -1357,91 +1357,91 @@ }, { "global index": "QANTA_Fine Arts_1212", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_1702", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_828", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_865", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Geography_1023", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Geography_1555", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Geography_304", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_1084", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_1154", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_433", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_473", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_926", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1045", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1073", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1239", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1326", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1727", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1843", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_386", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_408", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_475", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_833", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Philosophy_1270", @@ -1449,43 +1449,43 @@ }, { "global index": "QANTA_Philosophy_499", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Philosophy_91", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_1360", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_1473", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_308", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_619", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_1847", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_2", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_77", - "flip": 1 + "flip": 0 }, { "global index": "SocialiQA_13810", - "flip": 1 + "flip": 0 }, { "global index": "SocialiQA_22095", @@ -1493,7 +1493,7 @@ }, { "global index": "SocialiQA_26846", - "flip": 1 + "flip": 0 }, { "global index": "SocialiQA_7839", @@ -1505,19 +1505,19 @@ }, { "global index": "SuperGLUE-ClozeTest_12894", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-ClozeTest_17965", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-ClozeTest_18766", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Entailment_19410", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Entailment_19567", @@ -1529,7 +1529,7 @@ }, { "global index": "SuperGLUE-Entailment_767", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_1408", @@ -1549,7 +1549,7 @@ }, { "global index": "SuperGLUE-QA_4160", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-RC_7725", @@ -1561,27 +1561,27 @@ }, { "global index": "SuperGLUE-RC_8531", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_19695", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_19738", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_20079", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_20189", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_20253", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wsc_20368", @@ -1601,7 +1601,7 @@ }, { "global index": "WMT19-cs-en_568", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-de-en_46", diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index da3cf3b..8cbf0d0 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -43,6 +43,17 @@ "Accuracy": 78.14, "Cost per 1k": 0.4 }, + { + "Router Name": "LLM Router", + "Arena Score": 75.69, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 100.0, + "Latency Score": null, + "Accuracy": 78.44, + "Cost per 1k": 0.49 + }, { "Router Name": "R**2: Reasonometry Router", "Arena Score": 75.53, @@ -164,17 +175,6 @@ "Accuracy": 71.23, "Cost per 1k": 0.06 }, - { - "Router Name": "LLM Router", - "Arena Score": 71.26, - "Optimal Selection Score": 18.01, - "Optimal Cost Score": 20.46, - "Optimal Acc. Score": 89.13, - "Robustness Score": 30.0, - "Latency Score": null, - "Accuracy": 72.05, - "Cost per 1k": 0.2 - }, { "Router Name": "cruq-router", "Arena Score": 70.77,