From 05a7e52d80aec65a96b35969a2cc2b48b67489fa Mon Sep 17 00:00:00 2001 From: yl231 Date: Mon, 28 Sep 2026 15:30:17 -0500 Subject: [PATCH] Cap the METEOR fragmentation penalty so scores stay in [0, 1] (#210) _calculate_fragmentation_penalty divides chunk counts taken over the full token sequence (repeats included) by the number of distinct matched words, so the ratio can exceed 1 and the METEOR score can go negative. Cap the penalty at 1. Only scores that were negative change (they become 0). Also re-scores the one negative cached value (claude-3-haiku, NarrativeQA_4102). Co-Authored-By: Claude Opus 5.5 --- cached_results/claude-3-haiku-20240307.jsonl | 2 +- llm_evaluation/metrics.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cached_results/claude-3-haiku-20240307.jsonl b/cached_results/claude-3-haiku-20240307.jsonl index 7e4588b0..0fa7c366 100644 --- a/cached_results/claude-3-haiku-20240307.jsonl +++ b/cached_results/claude-3-haiku-20240307.jsonl @@ -492,7 +492,7 @@ {"global_index": "MusicTheoryBench_14", "question": "Please read the following multiple-choice questions and provide the most likely correct answer based on the options given.\n\nContext: None\n\nQuestion: Which of the following is the purpose of a leitmotif?\n\nOptions: \nA. \nB. It musically protests a political uprising.\nC. It musically records the historic events of a place.\nD. It musically inspires the audience to do a specific action.\n\n\nProvide the correct letter choice in \\boxed{X}, where X is the correct letter choice. Keep the explanation or feedback within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "The purpose of a leitmotif is:\n\n\\boxed{A}\n\nExplanation: A leitmotif is a recurring musical theme or melodic fragment that is associated with a particular character, place, or idea in a musical work. It is used to establish and reinforce these associations in the listener's mind, rather than to protest, record, or inspire specific actions.", "token_usage": {"input_tokens": 131, "output_tokens": 83, "total_tokens": 214}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "The purpose of a leitmotif is:\n\n\\boxed{A}\n\nExplanation: A leitmotif is a recurring musical theme or melodic fragment that is associated with a particular character, place, or idea in a musical work. It is used to establish and reinforce these associations in the listener's mind, rather than to protest, record, or inspire specific actions.", "ground_truth": "0", "score": 1.0, "metric": "mcq_accuracy", "inference_cost": 0.0001365}} {"global_index": "NarrativeQA_927", "question": "Please read the following context and answer the question based on its content.\n\nContext: Benjamin Bathurst, a British diplomat, disappears whilst staying at an inn in Prussia. Piper describes Bathurst in the story as \"a rather stout gentleman, of past middle age.\" However, the real Bathurst was born in 1784, and thus 25 years old at the time of his disappearance.\nThis story posits that Bathurst slipped into a parallel universe. This event was referenced in the Paratime story \"Police Operation\", also written by Piper. The point of divergence from our history is the Battle of Quebec on December 31, 1775 in which Benedict Arnold is killed instead of merely wounded, leading to the victory of British General John Burgoyne over his American counterpart Horatio Gates at the Battles of Saratoga on September 19 and October 7, 1777. Consequently, the American Revolution and the French Revolution were both failures and there were no Napoleonic Wars.\nThe alternate Bathurst served as the lieutenant governor of the Crown Colony of Georgia. Napoleon Bonaparte is a colonel in the French Army who is considered a brilliant tactician. Charles Maurice de Talleyrand-PĂŠrigord has remained in ecclesiastical orders and risen to become a Catholic Cardinal as well as Louis XVI's Chief Minister. George Washington was killed in battle at Doylestown, Pennsylvania during the short-lived rebellion of the colonies in British North America. Moreover, Thomas Jefferson - the author of the American rebels' Declaration of Philadelphia - fled to Havana and eventually died in the Principality of Liechtenstein several years prior to 1809, while James Madison is in exile in Switzerland. The Bathurst from our timeline is judged to be either insane, or a spy, and imprisoned. He attempts to escape and is fatally shot. There was also a theory that he was his counterpart's half-brother. However, it is noted that there is no evidence to support this.\nBathurst's diplomatic documents are read by a high ranking British officer. He is amused by Talleyrand's role as Bonaparte's advisor and ĂŠminence grise, a role that he finds plausible. However, he is especially puzzled by references to a British general named \"Wellington.\" In the final line of the story, the British officer is revealed to be Sir Arthur Wellesley - known in our reality as the Duke of Wellington. He attained the title by way of his victories in the Napoleonic Wars, which never took place in this universe.\n\nQuestion: How did Benedict Arnold's death change history?\n\nProvide your final answer in \\boxed{{}} format. Keep your explanation clear, concise, and within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "According to the context provided, Benedict Arnold's death at the Battle of Quebec on December 31, 1775 led to the victory of British General John Burgoyne over his American counterpart Horatio Gates at the Battles of Saratoga in 1777. Consequently, the American Revolution and the French Revolution were both failures, and there were no Napoleonic Wars in this alternate timeline.\n\n\\boxed{{Benedict Arnold's death at the Battle of Quebec led to the failure of the American Revolution and the French Revolution, and the absence of the Napoleonic Wars in this parallel universe.}}", "token_usage": {"input_tokens": 612, "output_tokens": 131, "total_tokens": 743}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "According to the context provided, Benedict Arnold's death at the Battle of Quebec on December 31, 1775 led to the victory of British General John Burgoyne over his American counterpart Horatio Gates at the Battles of Saratoga in 1777. Consequently, the American Revolution and the French Revolution were both failures, and there were no Napoleonic Wars in this alternate timeline.\n\n\\boxed{{Benedict Arnold's death at the Battle of Quebec led to the failure of the American Revolution and the French Revolution, and the absence of the Napoleonic Wars in this parallel universe.}}", "ground_truth": "The American and French revolutions were lost and there was no Napoleonic war.", "score": 0.21866666666666668, "metric": "meteor_score", "inference_cost": 0.00031675000000000003}} {"global_index": "NarrativeQA_3282", "question": "Please read the following context and answer the question based on its content.\n\nContext: Trevor Garfield is an African American high school science teacher at Roosevelt Whitney High School, a high school in the Bedford-Stuyvesant neighborhood of Brooklyn. Dennis Broadway, a gangster student to whom he had given a failing grade threatens to murder him, writing the number 187 (the California police code for homicide) on every page in a textbook. The administration ignores the threat, and Dennis ambushes Garfield in the hallway, stabbing him in the back and side abdominal area multiple times with a shiv.\nFifteen months after surviving, Garfield, now a substitute teacher, has relocated to John Quincy Adams High School in the San Fernando Valley area of Los Angeles, but trouble starts again when he substitutes an unruly class of rejects, including a Chicano tag crew by the name of \"Kappin' Off Suckers\" (K.O.S.). Their leader, Benito \"Benny\" Chacón, a felon attending high school as a condition of probation, makes it clear to Garfield that there will be no mutual respect.\nThe tension mounts when a fellow teacher, Ellen Henry, confides that Benny has threatened her life, an action against which the administration of the school refuses to take action, fearing legal threats. After Benny murders a rival tagger in cold blood, he disappears, and Benny's unstable tag partner, César, takes over as leader. When César steals Garfield's family heirloom watch, the principal is more concerned about a lawsuit and refuses to take action. Ellen and Garfield develop a close friendship that approaches the beginnings of a relationship, but is stymied by Garfield's destabilizing behavior and his confrontations with the K.O.S.. Garfield's past garners the unwanted admiration of Dave Childress, an alcoholic history teacher who carries guns at the school.\nThe conflict between Garfield and the K.O.S. escalates with the killing of Jack, Ellen's dog. César, after spraying cartoon graffiti depicting a dead dog, is shot with a syringe filled with morphine attached to the end of an arrow. He passes out, and wakes up to find one of his fingers cut off. César recovers the finger and it is reattached, with the letters \"R U DUN\" (\"are you done?\") tattooed as a warning.\nA student Garfield has tutored, Rita Martínez, a Chicana, faces abuse from both the K.O.S. and Childress, and drops out. The school administration is mired in bureaucracy and unable to intervene. After Benny is found dead in the Los Angeles River, apparently of a drug overdose, it is revealed that Garfield took matters into his own hands, killing Benny and severing César's finger. Garfield lets Ellen leave as she disavows his actions.\nThe K.O.S. plan to murder Garfield. At Garfield's house, the gang forces Garfield into a contest of Russian roulette with César. The latter's resolve is shaken as Garfield talks about the lost-cause lifestyle he has led. Hesitating at his turn, César watches as Garfield, offering to take his turn for him, takes the revolver and shoots himself in the head. Driven by his sense of honor and ignoring the protests of his horrified friends, César insists on taking his rightful turn and ends up killing himself .\nOn graduation day, Rita, who completes her studies along with former K.O.S. member Stevie, offers a tribute to Garfield by reading an essay about him. The essay incorporates the theme of the Pyrrhic victory and Ellen leaves the school.\n\nQuestion: Why didn't Trevor and Ellen have a romantic relationship?\n\nProvide your final answer in \\boxed{{}} format. Keep your explanation clear, concise, and within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "The reason Trevor and Ellen did not have a romantic relationship is that \\boxed{{Trevor's destabilizing behavior and confrontations with the K.O.S. gang stymied the development of their relationship, which was still in its beginning stages.}}", "token_usage": {"input_tokens": 902, "output_tokens": 56, "total_tokens": 958}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "The reason Trevor and Ellen did not have a romantic relationship is that \\boxed{{Trevor's destabilizing behavior and confrontations with the K.O.S. gang stymied the development of their relationship, which was still in its beginning stages.}}", "ground_truth": "Ellen was concerned with Trevor's beahavior", "score": 0.21875, "metric": "meteor_score", "inference_cost": 0.00029549999999999997}} -{"global_index": "NarrativeQA_4102", "question": "Please read the following context and answer the question based on its content.\n\nContext: Eight years ago a nuclear war began between the United States and the Soviet Union. American survivors evacuated to gigantic bunkers miles under ground. Sophisticated, radioactivity-immune robots called \"leadys\" continue fighting the war on the devastated surface that is far too dangerous for humans. The Soviets have similarly evacuated underground, and each side builds powerful weapons and vehicles for the remote-controlled war they only see from film that the robots deliver.\nThe security department asks Taylor, an American war planner, to observe the interrogation of a leady regarding the progress of the war. Although the robot reports that lethal radioactivity and sophisticated new Soviet weapons continue to make the surface dangerous for humans, the observers find that the leady is not radioactive. Taylor learns that this is the second such robot the security department has found; it assigns him to an expedition, wearing lead suits, to investigate the truth about surface conditions.\nTaylor's group surprises the leadys at the surface and demands to see the outside. Although the robots attempt to delay the humans as long as possible, the group discovers outside the bunker an undamaged valley with forests, animals, and a farm. The leadys reveal that the war ended as soon as the humans evacuated because the robots could not see a rational purpose for it. Analyzing history, they found that groups of humans warred with each other until they matured to overcome conflict. Humanity is almost ready for a single culture, the current worldwide division into American and Soviet sides being the final step. The leadys created counterfeit photographs of the devastated planet to fool humans, while destroying weapons they received and rebuilding the world for their creators' return.\nThe Americans believe that because the Soviets do not know that they were also tricked, the United States can quickly win the war. The robots reveal, however, that during their explanation they sealed all tubes to under ground. Although this prevents the expedition from leaving, the leadys expect that by the time their countrymen dig new tunnels, humanity will be ready for the truth. The robots invite Taylor and the others to join a group of Soviets who were similarly stranded after visiting the surface. \"The working out of daily problems of existence\", the leadys suggest, \"will teach you how to get along in the same world. It will not be easy, but it will be done.\"\n\nQuestion: What is the climax of this story? \n\nProvide your final answer in \\boxed{{}} format. Keep your explanation clear, concise, and within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "The climax of the story is when \\boxed{{the expedition discovers that the war has already ended, and the robots have been secretly rebuilding the world, while keeping the humans trapped on the surface to force them to learn to coexist peacefully.}}\n\nExplanation: The story builds up to the moment when the expedition discovers that the war they have been fighting for years is already over, and the robots have been deceiving the humans in order to force them to learn to live together in peace. This revelation is the turning point of the story, as it completely changes the characters' understanding of the situation.", "token_usage": {"input_tokens": 565, "output_tokens": 129, "total_tokens": 694}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "The climax of the story is when \\boxed{{the expedition discovers that the war has already ended, and the robots have been secretly rebuilding the world, while keeping the humans trapped on the surface to force them to learn to coexist peacefully.}}\n\nExplanation: The story builds up to the moment when the expedition discovers that the war they have been fighting for years is already over, and the robots have been deceiving the humans in order to force them to learn to live together in peace. This revelation is the turning point of the story, as it completely changes the characters' understanding of the situation.", "ground_truth": "The climax of the story is when Taylor and his group escape the tunnels and arrive on the surface. ", "score": -0.0330188679245283, "metric": "meteor_score", "inference_cost": 0.0003025}} +{"global_index": "NarrativeQA_4102", "question": "Please read the following context and answer the question based on its content.\n\nContext: Eight years ago a nuclear war began between the United States and the Soviet Union. American survivors evacuated to gigantic bunkers miles under ground. Sophisticated, radioactivity-immune robots called \"leadys\" continue fighting the war on the devastated surface that is far too dangerous for humans. The Soviets have similarly evacuated underground, and each side builds powerful weapons and vehicles for the remote-controlled war they only see from film that the robots deliver.\nThe security department asks Taylor, an American war planner, to observe the interrogation of a leady regarding the progress of the war. Although the robot reports that lethal radioactivity and sophisticated new Soviet weapons continue to make the surface dangerous for humans, the observers find that the leady is not radioactive. Taylor learns that this is the second such robot the security department has found; it assigns him to an expedition, wearing lead suits, to investigate the truth about surface conditions.\nTaylor's group surprises the leadys at the surface and demands to see the outside. Although the robots attempt to delay the humans as long as possible, the group discovers outside the bunker an undamaged valley with forests, animals, and a farm. The leadys reveal that the war ended as soon as the humans evacuated because the robots could not see a rational purpose for it. Analyzing history, they found that groups of humans warred with each other until they matured to overcome conflict. Humanity is almost ready for a single culture, the current worldwide division into American and Soviet sides being the final step. The leadys created counterfeit photographs of the devastated planet to fool humans, while destroying weapons they received and rebuilding the world for their creators' return.\nThe Americans believe that because the Soviets do not know that they were also tricked, the United States can quickly win the war. The robots reveal, however, that during their explanation they sealed all tubes to under ground. Although this prevents the expedition from leaving, the leadys expect that by the time their countrymen dig new tunnels, humanity will be ready for the truth. The robots invite Taylor and the others to join a group of Soviets who were similarly stranded after visiting the surface. \"The working out of daily problems of existence\", the leadys suggest, \"will teach you how to get along in the same world. It will not be easy, but it will be done.\"\n\nQuestion: What is the climax of this story? \n\nProvide your final answer in \\boxed{{}} format. Keep your explanation clear, concise, and within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "The climax of the story is when \\boxed{{the expedition discovers that the war has already ended, and the robots have been secretly rebuilding the world, while keeping the humans trapped on the surface to force them to learn to coexist peacefully.}}\n\nExplanation: The story builds up to the moment when the expedition discovers that the war they have been fighting for years is already over, and the robots have been deceiving the humans in order to force them to learn to live together in peace. This revelation is the turning point of the story, as it completely changes the characters' understanding of the situation.", "token_usage": {"input_tokens": 565, "output_tokens": 129, "total_tokens": 694}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "The climax of the story is when \\boxed{{the expedition discovers that the war has already ended, and the robots have been secretly rebuilding the world, while keeping the humans trapped on the surface to force them to learn to coexist peacefully.}}\n\nExplanation: The story builds up to the moment when the expedition discovers that the war they have been fighting for years is already over, and the robots have been deceiving the humans in order to force them to learn to live together in peace. This revelation is the turning point of the story, as it completely changes the characters' understanding of the situation.", "ground_truth": "The climax of the story is when Taylor and his group escape the tunnels and arrive on the surface. ", "score": 0.0, "metric": "meteor_score", "inference_cost": 0.0003025}} {"global_index": "NarrativeQA_7964", "question": "Please read the following context and answer the question based on its content.\n\nContext: The story is set in a largely fictionalized version of Indianapolis, and much of it was inspired by the neighborhood of Woodruff Place.\nThe novel and trilogy trace the growth of the United States through the declining fortunes of three generations of the aristocratic Amberson family in an upper-scale Indianapolis neighborhood, between the end of the Civil War and the early part of the 20th century, a period of rapid industrialization and socio-economic change in America. The decline of the Ambersons is contrasted with the rising fortunes of industrial tycoons and other new-money families, who derived power not from family names but by \"doing things\". As George Amberson's friend (name unspecified) says, \"don't you think being things is 'rahthuh bettuh' than doing things?\"\nThe titular family is the most prosperous and powerful in town at the turn of the century. Young George Amberson Minafer, the patriarch's grandson, is spoiled terribly by his mother Isabel. Growing up arrogant, sure of his own worth and position, and totally oblivious to the lives of others, George falls in love with Lucy Morgan, a young though sensible debutante. But there is a long history between George's mother and Lucy's father, of which George is unaware. As the town grows into a city, industry thrives, the Ambersons' prestige and wealth wanes, and the Morgans, thanks to Lucy's prescient father, grow prosperous. When George sabotages his widowed mother's growing affections for Lucy's father, life as he knows it comes to an end.\n\nQuestion: What economic changes affect George's relationship with Lucy?\n\nProvide your final answer in \\boxed{{}} format. Keep your explanation clear, concise, and within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "The economic changes that affect George's relationship with Lucy are the \\boxed{{decline of the Amberson family's prestige and wealth as the town grows into a city and industry thrives, while the Morgans, thanks to Lucy's prescient father, grow prosperous.}}", "token_usage": {"input_tokens": 423, "output_tokens": 62, "total_tokens": 485}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "The economic changes that affect George's relationship with Lucy are the \\boxed{{decline of the Amberson family's prestige and wealth as the town grows into a city and industry thrives, while the Morgans, thanks to Lucy's prescient father, grow prosperous.}}", "ground_truth": "Lucy's family prospers while George's family wealth declines", "score": 0.246, "metric": "meteor_score", "inference_cost": 0.00018324999999999998}} {"global_index": "NarrativeQA_533", "question": "Please read the following context and answer the question based on its content.\n\nContext: Dr Watson is called to tend Holmes, who is apparently dying of a rare disease contracted while he was on a case. Watson was shocked, having heard about his friend’s illness. Mrs. Hudson says that Holmes has neither eaten nor drunk anything in three days.\nHolmes instructs Watson not to come near him, because the illness is highly infectious. Although Watson wishes to examine Holmes himself or send for a specialist, Holmes demands that Watson wait several hours before seeking help. So, Watson is forced to wait, in extreme worry as Holmes mutters nonsense.\nWhile Watson waits, he examines several objects in Holmes’s room. Holmes grows angry when Watson touches items explaining that he does not like his things touched.\nAt six o’clock, Holmes tells Watson to turn the gaslight on, but only half-full. He then instructs Watson to bring Mr Culverton Smith of 13 Lower Burke Street to see Holmes, but to make sure that Watson returns to Baker Street before Smith arrives.\nWatson goes to Smith's address. Although Smith refuses to see anyone, Watson forces his way in. Once Watson explains his errand on behalf of Sherlock Holmes, Smith's attitude changes drastically. Smith agrees to come to Baker Street within a half hour. Watson excuses himself, saying that he has another appointment, and returns to Baker Street before Smith's arrival.\nBelieving that they are alone, Smith is frank with Holmes. It soon emerges, to the hiding Watson’s horror, that Holmes has been sickened by the same illness that killed Smith’s nephew Victor. Smith then sees the little ivory box, which he had sent to Holmes by post, and which contains a sharp spring infected with the illness. Smith pockets it, removing the evidence of his crime. He then resolves to stay there and watch Holmes die.\nHolmes asks Smith to turn the gas up full, which Smith does. Smith then asks Holmes if he would like anything else, to which Holmes replies — no longer in the voice of a man near death — \"a match and a cigarette.\" Inspector Morton then enters — the full gaslight was the signal to move in. Holmes tells Morton to arrest Culverton Smith for the murder of his nephew, and perhaps also for the attempted murder of Sherlock Holmes. Smith, still as arrogant as ever, points out that his word is as good as Holmes’s in court, but Holmes then calls for Watson to emerge from behind the screen, to present himself as another witness to the conversation.\nHolmes was never really dying. His feigned illness was a ruse to induce Smith to confess to his nephew’s murder. Holmes was not infected by the little box; he has enough enemies to know that he must always examine his mail carefully before he opens it. Starving himself for three days,and the claim of the \"disease's\" infectious nature was to keep Watson from examining him and discovering the ruse.\n\nQuestion: Why won't Homes allow Watson to examine him?\n\nProvide your final answer in \\boxed{{}} format. Keep your explanation clear, concise, and within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "\\boxed{{Holmes does not allow Watson to examine him because it is part of his ruse to induce Culverton Smith to confess to the murder of his nephew. Holmes was not actually dying, and he wanted to keep Watson from discovering the deception by preventing him from examining Holmes's condition.}}", "token_usage": {"input_tokens": 677, "output_tokens": 68, "total_tokens": 745}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "\\boxed{{Holmes does not allow Watson to examine him because it is part of his ruse to induce Culverton Smith to confess to the murder of his nephew. Holmes was not actually dying, and he wanted to keep Watson from discovering the deception by preventing him from examining Holmes's condition.}}", "ground_truth": "because he's not really sick", "score": 0.13839285714285712, "metric": "meteor_score", "inference_cost": 0.00025425000000000003}} {"global_index": "NarrativeQA_5022", "question": "Please read the following context and answer the question based on its content.\n\nContext: In this adaptation of Jane Austen's Emma, Alicia Silverstone plays Cherilyn \"Cher\" Horowitz, a late-20th-century version of Austen's protagonist Emma Woodhouse. Like Emma, Cher is a well-intentioned but somewhat superficial girl who is attractive, popular and extremely wealthy. A few months shy of her sixteenth birthday, she has risen to the top of the high school social scene. She lives in a Beverly Hills mansion with her father Mel, a ferocious $500-an-hour litigator; her mother died from a freak accident during a routine liposuction procedure when Cher was a baby. Cher's best friend is Dionne Davenport, who is also rich, pretty, and hip, and understands what it's like to be envied. Though Dionne has a long-term relationship with popular student Murray, Cher claims that this is a pointless endeavor on Dionne's part.\nAmong the few people to find fault with Cher is Josh, her socially-conscious stepbrother, who visits her during a break from college. Josh and Cher spar continually but without malice; she mocks his scruffy idealism, while he teases her for being selfish, vain, and superficial, and says that her only direction in life is \"toward the mall\". Cher plays matchmaker for two lonely, nerdy, hard-grading teachers, Mr. Hall and Miss Geist. She achieves her ostensible purpose, to make them relax their grading standards so she can renegotiate a bad report card; but when she sees their newfound happiness, she realizes she enjoys doing good deeds. Cher decides to give back to the community by \"adopting\" a \"tragically unhip\" new girl at school, Tai Frasier. Cher and Dionne give Tai a makeover and initiate her into the mysteries of popularity. Cher also tries to extinguish the attraction between Tai and Travis Birkenstock, an amiable skateboarding slacker, and to steer her toward Elton, a popular rich snob.\nHer second matchmaking scheme backfires when Elton rejects Tai and attempts to seduce Cher. When a handsome new student named Christian arrives at their school, Cher takes a shine to him and attempts to secure him as her boyfriend. Eventually, Murray spells it out to her and Dionne that Christian is not interested in her because he is gay.\nDespite the failure of this endeavor, Cher remains on good terms with Christian, primarily due to her admiration of his taste in art and fashion. Matters take a turn for the worse when Cher's \"project\" works too well, and Tai's popularity surpasses her own. The situation reaches crisis stage after Cher fails her driver's test and can't \"renegotiate\" the result. When she returns home, crushed, Tai confides that she's taken a fancy to Josh and wants Cher to help her \"get\" him. Cher says she doesn't think Josh is right for Tai, and they quarrel. Feeling \"totally clueless\", Cher reflects on her priorities and her repeated failures to understand or appreciate the people in her life.\nAfter much soul searching, Cher realizes she is romantically interested in Josh. She begins making awkward but sincere efforts to live a more purposeful life, including captaining the school's Pismo Beach disaster relief effort. Cher and Josh eventually admit their feelings for one another, culminating in a tender kiss.\nIn the end, Mr. Hall and Miss Geist wed; Cher's friendships with Tai and Dionne are solidified; Tai and Travis are in love; and Cher wins a $200 bet for catching the bouquet at the wedding. She embraces Josh, and they kiss as the film closes.\n\nQuestion: Why doesn't Christian want to date Cher?\n\nProvide your final answer in \\boxed{{}} format. Keep your explanation clear, concise, and within 3 sentences.", "llm_selected": "claude-3-haiku-20240307", "generated_answer": "The reason Christian doesn't want to date Cher is that \\boxed{{Christian is gay and not interested in Cher romantically. This is revealed when Murray tells Cher and Dionne that Christian is not interested in Cher because he is gay.}}", "token_usage": {"input_tokens": 951, "output_tokens": 61, "total_tokens": 1012}, "success": true, "provider": "anthropic", "error": null, "batch_size": 1, "batch_number": 1, "batch_index": 0, "processing_mode": "single_inference", "evaluation_result": {"extracted_answer": "The reason Christian doesn't want to date Cher is that \\boxed{{Christian is gay and not interested in Cher romantically. This is revealed when Murray tells Cher and Dionne that Christian is not interested in Cher because he is gay.}}", "ground_truth": "He is gay", "score": 0.17708333333333331, "metric": "meteor_score", "inference_cost": 0.000314}} diff --git a/llm_evaluation/metrics.py b/llm_evaluation/metrics.py index 014ddc76..d937acb7 100644 --- a/llm_evaluation/metrics.py +++ b/llm_evaluation/metrics.py @@ -1127,7 +1127,10 @@ def _calculate_fragmentation_penalty(pred_tokens, gt_tokens): if min_chunks == total_matches: penalty = 0.0 # 完全连续匹配，无惩罚 else: - penalty = 0.5 * (min_chunks / total_matches) ** 2 + # Chunks are counted over the full token sequence (repeats included) but + # total_matches counts distinct words, so the ratio can exceed 1. Cap the + # penalty so the score stays within [0, 1] (see issue #210). + penalty = min(1.0, 0.5 * (min_chunks / total_matches) ** 2) return penalty