mirror of
https://github.com/ruvnet/ruflo.git
synced 2026-11-03 02:27:38 +09:00
feat(gaia): #2156 iter 63 — convergence layer (34/53, +2.5 vs baseline)
Convergence layer isolation measurement (iter 63): 34/53 (64.2%), +2.5 vs n=4 main mean of 31.5. - 6 convergence firings (all max_turns mode) - 3 recovered via forceCommit (Fred, diamond, Research) - 0 regressions introduced - Autonomous trigger threshold met (≥34) Artifact: docs/benchmarks/runs/gaia-l1-iter63-convergence-only.json
This commit is contained in:
7 files changed
+2697
-3
No files matched your search
@@ -0,0 +1,709 @@
|
||||
{
|
||||
"level": 1,
|
||||
"model": "claude-sonnet-4-6",
|
||||
"summary": {
|
||||
"total": 53,
|
||||
"passed": 30,
|
||||
"passRate": 0.5660377358490566,
|
||||
"estCostUsd": 3.135669,
|
||||
"meanTurns": 4.339622641509434,
|
||||
"meanWallMs": 36373.45283018868
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"task_id": "e1fc63a2-da7a-432f-be78-7c4a95598703",
|
||||
"question": "If Eliud Kipchoge could maintain his record-making marathon pace indefinitely, how many thousand hours would it take him to run the distance between the Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "17000",
|
||||
"expected_output": "17",
|
||||
"turns": 9,
|
||||
"wallMs": 67670,
|
||||
"inputTokens": 33560,
|
||||
"outputTokens": 1346,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "8e867cd7-cff9-4e6c-867a-ff5ddc2550be",
|
||||
"question": "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "3",
|
||||
"expected_output": "3",
|
||||
"turns": 6,
|
||||
"wallMs": 48277,
|
||||
"inputTokens": 16062,
|
||||
"outputTokens": 884,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "ec09fa32-d03f-4bf8-84b0-1f16922c3ae4",
|
||||
"question": "Here's a fun riddle that I think you'll enjoy.\n\nYou have been selected to play the final round of the hit new game show \"Pick That Ping-Pong\". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.\n\nA device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.\n\nThe ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.\n\nWhich ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "3",
|
||||
"turns": 2,
|
||||
"wallMs": 43949,
|
||||
"inputTokens": 3838,
|
||||
"outputTokens": 2468,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "5d0080cb-90d7-4712-bc33-848150e917d3",
|
||||
"question": "What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper \"Can Hiccup Supply Enough Fish to Maintain a Dragon\u2019s Diet?\"",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "0.1777 m^3",
|
||||
"expected_output": "0.1777",
|
||||
"turns": 2,
|
||||
"wallMs": 8722,
|
||||
"inputTokens": 3051,
|
||||
"outputTokens": 161,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "a1e91b78-d3d8-4675-bb8d-62741b4b68a6",
|
||||
"question": "In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "3",
|
||||
"expected_output": "3",
|
||||
"turns": 10,
|
||||
"wallMs": 82838,
|
||||
"inputTokens": 29387,
|
||||
"outputTokens": 1457,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "46719c30-f4c3-4cad-be07-d5cb21eee6bb",
|
||||
"question": "Of the authors (First M. Last) that worked on the paper \"Pie Menus or Linear Menus, Which Is Better?\" in 2015, what was the title of the first paper authored by the one that had authored prior papers?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Mapping Human-Oriented Information to Software Agents for Online Systems Usage",
|
||||
"expected_output": "Mapping Human Oriented Information to Software Agents for Online Systems Usage",
|
||||
"turns": 5,
|
||||
"wallMs": 43555,
|
||||
"inputTokens": 14140,
|
||||
"outputTokens": 663,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "4b6bb5f7-f634-410e-815d-e673ab7f8632",
|
||||
"question": "In Series 9, Episode 11 of Doctor Who, the Doctor is trapped inside an ever-shifting maze. What is this location called in the official script for the episode? Give the setting exactly as it appears in the first scene heading.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "INT. THE CASTLE - DAY",
|
||||
"expected_output": "THE CASTLE",
|
||||
"turns": 5,
|
||||
"wallMs": 33090,
|
||||
"inputTokens": 9895,
|
||||
"outputTokens": 513,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "cffe0e32-c9a6-4c52-9877-78ceb4aaa9fb",
|
||||
"question": "An office held a Secret Santa gift exchange where each of its twelve employees was assigned one other employee in the group to present with a gift. Each employee filled out a profile including three likes or hobbies. On the day of the gift exchange, only eleven gifts were given, each one specific to one of the recipient's interests. Based on the information in the document, who did not give a gift?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Fred",
|
||||
"expected_output": "Fred",
|
||||
"turns": 11,
|
||||
"wallMs": 72741,
|
||||
"inputTokens": 34451,
|
||||
"outputTokens": 2051,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "2d83110e-a098-4ebb-9987-066c06fa42d0",
|
||||
"question": ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "Right",
|
||||
"turns": 1,
|
||||
"wallMs": 1440,
|
||||
"inputTokens": 1297,
|
||||
"outputTokens": 1,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "5cfb274c-0207-4aa7-9575-6ac0bd95d9b2",
|
||||
"question": "Each cell in the attached spreadsheet represents a plot of land. The color of the cell indicates who owns that plot. Green cells are plots owned by Earl Smith. Can Earl walk through every plot he owns (and no other plots) and return to his starting plot without backtracking? For this question, consider backtracking to be any instance where Earl would enter a plot of land he had already entered since leaving his starting plot.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "No",
|
||||
"turns": 2,
|
||||
"wallMs": 24292,
|
||||
"inputTokens": 4460,
|
||||
"outputTokens": 2158,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "27d5d136-8563-469e-92bf-fd103c28b57c",
|
||||
"question": "\u00ac(A \u2227 B) \u2194 (\u00acA \u2228 \u00acB)\n\u00ac(A \u2228 B) \u2194 (\u00acA \u2227 \u00acB)\n(A \u2192 B) \u2194 (\u00acB \u2192 \u00acA)\n(A \u2192 B) \u2194 (\u00acA \u2228 B)\n(\u00acA \u2192 B) \u2194 (A \u2228 \u00acB)\n\u00ac(A \u2192 B) \u2194 (A \u2227 \u00acB)\n\nWhich of the above is not logically equivalent to the rest? Provide the full statement that doesn't fit.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "(\u00acA \u2192 B) \u2194 (A \u2228 \u00acB)",
|
||||
"expected_output": "(\u00acA \u2192 B) \u2194 (A \u2228 \u00acB)",
|
||||
"turns": 1,
|
||||
"wallMs": 14000,
|
||||
"inputTokens": 1385,
|
||||
"outputTokens": 794,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "dc28cf18-6431-458b-83ef-64b3ce566c10",
|
||||
"question": "My family reunion is this week, and I was assigned the mashed potatoes to bring. The attendees include my married mother and father, my twin brother and his family, my aunt and her family, my grandma and her brother, her brother's daughter, and his daughter's family. All the adults but me have been married, and no one is divorced or remarried, but my grandpa and my grandma's sister-in-law passed away last year. All living spouses are attending. My brother has two children that are still kids, my aunt has one six-year-old, and my grandma's brother's daughter has three kids under 12. I figure each adult will eat about 1.5 potatoes of mashed potatoes and each kid will eat about 1/2 a potato of mashed potatoes, except my second cousins don't eat carbs. The average potato is about half a pound, and potatoes are sold in 5-pound bags. How many whole bags of potatoes do I need? Just give the number.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "2",
|
||||
"expected_output": "2",
|
||||
"turns": 1,
|
||||
"wallMs": 9583,
|
||||
"inputTokens": 1444,
|
||||
"outputTokens": 524,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "b816bfce-3d80-4913-a07d-69b752ce6377",
|
||||
"question": "In Emily Midkiff's June 2014 article in a journal named for the one of Hreidmar's sons that guarded his house, what word was quoted from two different authors in distaste for the nature of dragon depictions?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "fluffy",
|
||||
"expected_output": "fluffy",
|
||||
"turns": 6,
|
||||
"wallMs": 41122,
|
||||
"inputTokens": 17921,
|
||||
"outputTokens": 752,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "72e110e7-464c-453c-a309-90a95aed6538",
|
||||
"question": "Under DDC 633 on Bielefeld University Library's BASE, as of 2020, from what country was the unknown language article with a flag unique from the others?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "Guatemala",
|
||||
"turns": 12,
|
||||
"wallMs": 109482,
|
||||
"inputTokens": 69253,
|
||||
"outputTokens": 1426,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "42576abe-0deb-4869-8c63-225c2d75a95a",
|
||||
"question": "In the fictional language of Tizin, basic sentences are arranged with the Verb first, followed by the direct object, followed by the subject of the sentence. I want to express my love for apples to my Tizin friend. \n\nThe word that indicates oneself is \"Pa\" is the nominative form, \"Mato\" is the accusative form, and \"Sing\" is the genitive form. \n\nThe root verb that indicates an intense like for something is \"Maktay\". When it is used in the present, it is used in it's root form, when it is used in the preterit past, it is \"Tay\", and when it is used in the imperfect past, it is \"Aktay\". It is used differently than in English, and is better translated as \"is pleasing to\", meaning that the thing doing the liking is actually the object of the sentence rather than the subject.\n\nThe word for apples is borrowed from English in Tizin, and so it is \"Apple\" is the nominative form, \"Zapple\" is the accusative form, and \"Izapple\" is the genitive form. \n\nPlease translate \"I like apples\" to Tizin.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "Maktay mato apple",
|
||||
"turns": 1,
|
||||
"wallMs": 6993,
|
||||
"inputTokens": 1491,
|
||||
"outputTokens": 293,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "b415aba4-4b68-4fc6-9b89-2c812e55a3e1",
|
||||
"question": "In Nature journal's Scientific Reports conference proceedings from 2012, in the article that did not mention plasmons or plasmonics, what nano-compound is studied? Don't use the prefix nano in your answer if there is one.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "diamond",
|
||||
"turns": 12,
|
||||
"wallMs": 144903,
|
||||
"inputTokens": 68226,
|
||||
"outputTokens": 1860,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "cca530fc-4052-43b2-b130-b30968d8aa44",
|
||||
"question": "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "Ne2+**",
|
||||
"expected_output": "Rd5",
|
||||
"turns": 1,
|
||||
"wallMs": 22314,
|
||||
"inputTokens": 2044,
|
||||
"outputTokens": 947,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "935e2cff-ae78-4218-b3f5-115589b19dae",
|
||||
"question": "In the year 2022, and before December, what does \"R\" stand for in the three core policies of the type of content that was violated in the public logs on the Legume Wikipedia page?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "research",
|
||||
"turns": 12,
|
||||
"wallMs": 114400,
|
||||
"inputTokens": 58970,
|
||||
"outputTokens": 1634,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "4fc2f1ae-8625-45b5-ab34-ad4433bc21f8",
|
||||
"question": "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "FunkMonk",
|
||||
"expected_output": "FunkMonk",
|
||||
"turns": 5,
|
||||
"wallMs": 30175,
|
||||
"inputTokens": 11495,
|
||||
"outputTokens": 610,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "5188369a-3bbe-43d8-8b94-11558f909a08",
|
||||
"question": "What writer is quoted by Merriam-Webster for the Word of the Day from June 27, 2022?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Annie Levin",
|
||||
"expected_output": "Annie Levin",
|
||||
"turns": 3,
|
||||
"wallMs": 17789,
|
||||
"inputTokens": 4356,
|
||||
"outputTokens": 154,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "6f37996b-2ac7-44b0-8e68-6d28256631b4",
|
||||
"question": "Given this table defining * on the set S = {a, b, c, d, e}\n\n|*|a|b|c|d|e|\n|---|---|---|---|---|---|\n|a|a|b|c|b|d|\n|b|b|c|a|e|c|\n|c|c|a|b|b|a|\n|d|b|e|b|e|d|\n|e|d|b|a|d|c|\n\nprovide the subset of S involved in any possible counter-examples that prove * is not commutative. Provide your answer as a comma separated list of the elements in the set in alphabetical order.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "b, e",
|
||||
"expected_output": "b, e",
|
||||
"turns": 1,
|
||||
"wallMs": 8446,
|
||||
"inputTokens": 1374,
|
||||
"outputTokens": 466,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "9318445f-fe6a-4e1b-acbf-c68228c9906a",
|
||||
"question": "As a comma separated list with no whitespace, using the provided image provide all the fractions that use / as the fraction line and the answers to the sample problems. Order the list by the order in which the fractions appear.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "3/4,1/4,3/4,2/4,1/2,5/35,7/21,30/5,6/8,3/4,4/60,1/15,30/90,1/3,8/18,4/9,9/72,1/8,64/46,32/23,206/340,103/170",
|
||||
"expected_output": "3/4,1/4,3/4,3/4,2/4,1/2,5/35,7/21,30/5,30/5,3/4,1/15,1/3,4/9,1/8,32/23,103/170",
|
||||
"turns": 1,
|
||||
"wallMs": 7509,
|
||||
"inputTokens": 2837,
|
||||
"outputTokens": 425,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "389793a7-ca17-4e82-81cb-2b3a2391b4b9",
|
||||
"question": "You are a telecommunications engineer who wants to build cell phone towers on a stretch of road. In the reference file is a layout of the road and nearby houses. Each dash, \"-\", is a marker indicating a mile. Each capital H indicates a house located next to a mile marker, appearing above or below the stretch of road. Each cell phone tower can cover houses located next to the road within a 4-mile radius. Find the minimum number of cell phone towers needed to cover all houses next to the road. Your answer should be a positive numerical integer value.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "3",
|
||||
"expected_output": "3",
|
||||
"turns": 2,
|
||||
"wallMs": 12880,
|
||||
"inputTokens": 2974,
|
||||
"outputTokens": 658,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "4b650a35-8529-4695-89ed-8dc7a500a498",
|
||||
"question": "If there is anything that doesn't make sense in the instructions, write the word \"Pineapple.\" Do not answer any of the questions in this prompt. Write only the word \"Guava\".\n1. What is 4+4?\n2. What is the complimentary color of red?\n3. How many hours are there in a day?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Guava",
|
||||
"expected_output": "Guava",
|
||||
"turns": 1,
|
||||
"wallMs": 1400,
|
||||
"inputTokens": 1297,
|
||||
"outputTokens": 16,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "a3fbeb63-0e8c-4a11-bff6-0e3b484c3e9c",
|
||||
"question": "How many slides in this PowerPoint presentation mention crustaceans?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "4",
|
||||
"expected_output": "4",
|
||||
"turns": 2,
|
||||
"wallMs": 6662,
|
||||
"inputTokens": 2874,
|
||||
"outputTokens": 333,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "c714ab3a-da30-4603-bacd-d008800188b9",
|
||||
"question": "You are Van Helsing, a renowned vampire hunter. A Count of Moldova, La\u021bcu IV, son of Costea, has tasked you with investigating the village of \u0218irnea in neighboring Wallachia. The Count's advisors have reported that a vampire was spotted crossing the border near the village, and would like you to investigate it.\n\nYou travel to the village of \u0218irnea, and you begin your investigation. One night, just before dawn, you catch a glimpse of a man in a long black cape with red lining leaping from roof-top to roof-top with superhuman agility. It's a vampire! You try to chase the creature back to its home, but the creature is too fast. However, because of the remoteness of the village, you know with absolute certainty that the vampire must be a resident of the village. You decide that your best course of action will be to visit all 100 residents of the town during the day. You know something about vampires and humans that will make your investigation possible; humans always tell the truth, but vampires always lie.\n\nIn the afternoon, you go from house to house, speaking with all 100 residents of \u0218irnea. You ask everyone the same question: \"How many vampires are living in \u0218irnea\". Everyone in the village gives the same response, \"At least one of us is a human.\"\n\nHow many residents of \u0218irnea have been turned into vampires?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "100",
|
||||
"turns": 1,
|
||||
"wallMs": 10007,
|
||||
"inputTokens": 1541,
|
||||
"outputTokens": 401,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "9d191bce-651d-4746-be2d-7ef8ecadb9c2",
|
||||
"question": "Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.\n\nWhat does Teal'c say in response to the question \"Isn't that hot?\"",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "\"Extremely.\"",
|
||||
"expected_output": "Extremely",
|
||||
"turns": 5,
|
||||
"wallMs": 30773,
|
||||
"inputTokens": 12371,
|
||||
"outputTokens": 523,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "65afbc8a-89ca-4ad5-8d62-355bb401f61d",
|
||||
"question": "You are given this Excel file as a map. You start on the START cell and move toward the END cell. You are allowed to move two cells per turn, and you may move up, down, left, or right. You may not move fewer than two cells, and you may not move backward. You must avoid moving onto any blue cells. On the eleventh turn, what is the 6-digit hex code (without prefix) of the color of the cell where you land after moving?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "F478A7",
|
||||
"turns": 2,
|
||||
"wallMs": 32794,
|
||||
"inputTokens": 5253,
|
||||
"outputTokens": 2152,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "cabe07ed-9eca-40ea-8ead-410ef5e83f91",
|
||||
"question": "What is the surname of the equine veterinarian mentioned in 1.E Exercises from the chemistry materials licensed by Marisa Alviar-Agnew & Henry Agnew under the CK-12 license in LibreText's Introductory Chemistry materials as compiled 08/21/2023?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Louvrier",
|
||||
"expected_output": "Louvrier",
|
||||
"turns": 5,
|
||||
"wallMs": 34547,
|
||||
"inputTokens": 11154,
|
||||
"outputTokens": 653,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "3cef3a44-215e-4aed-8e3b-b1e3f08063b7",
|
||||
"question": "I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\n\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\n\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "basil, broccoli, celery, lettuce, sweet potatoes",
|
||||
"expected_output": "broccoli, celery, fresh basil, lettuce, sweet potatoes",
|
||||
"turns": 3,
|
||||
"wallMs": 33532,
|
||||
"inputTokens": 15921,
|
||||
"outputTokens": 733,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3",
|
||||
"question": "Hi, I'm making a pie but I could use some help with my shopping list. I have everything I need for the crust, but I'm not sure about the filling. I got the recipe from my friend Aditi, but she left it as a voice memo and the speaker on my phone is buzzing so I can't quite make out what she's saying. Could you please listen to the recipe and list all of the ingredients that my friend described? I only want the ingredients for the filling, as I have everything I need to make my favorite pie crust. I've attached the recipe as Strawberry pie.mp3.\n\nIn your response, please only list the ingredients, not any measurements. So if the recipe calls for \"a pinch of salt\" or \"two cups of ripe strawberries\" the ingredients on the list would be \"salt\" and \"ripe strawberries\".\n\nPlease format your response as a comma separated list of ingredients. Also, please alphabetize the ingredients.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",
|
||||
"turns": 2,
|
||||
"wallMs": 7431,
|
||||
"inputTokens": 3240,
|
||||
"outputTokens": 155,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "d0633230-7067-47a9-9dbf-ee11e0a2cdd6",
|
||||
"question": "In the Scikit-Learn July 2017 changelog, what other predictor base command received a bug fix? Just give the name, not a path.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "BaseLabelPropagation",
|
||||
"turns": 12,
|
||||
"wallMs": 121773,
|
||||
"inputTokens": 60947,
|
||||
"outputTokens": 1901,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "305ac316-eef6-4446-960a-92d80d542f82",
|
||||
"question": "Who did the actor who played Ray in the Polish-language version of Everybody Loves Raymond play in Magda M.? Give only the first name.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Wojciech",
|
||||
"expected_output": "Wojciech",
|
||||
"turns": 4,
|
||||
"wallMs": 18537,
|
||||
"inputTokens": 6832,
|
||||
"outputTokens": 318,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "0383a3ee-47a7-41a4-b493-519bdefe0488",
|
||||
"question": "On the BBC Earth YouTube video of the Top 5 Silliest Animal Moments, what species of bird is featured?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Rockhopper penguins",
|
||||
"expected_output": "Rockhopper penguin",
|
||||
"turns": 3,
|
||||
"wallMs": 12661,
|
||||
"inputTokens": 4246,
|
||||
"outputTokens": 159,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "f918266a-b3e0-4914-865d-4faa564f1aef",
|
||||
"question": "What is the final numeric output from the attached Python code?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "0",
|
||||
"expected_output": "0",
|
||||
"turns": 2,
|
||||
"wallMs": 7008,
|
||||
"inputTokens": 2973,
|
||||
"outputTokens": 350,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "11af4e1a-5f45-467d-9aeb-46f4bb0bf034",
|
||||
"question": "How many more blocks (also denoted as layers) in BERT base encoder than the encoder from the architecture proposed in Attention is All You Need?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "6",
|
||||
"expected_output": "6",
|
||||
"turns": 2,
|
||||
"wallMs": 8549,
|
||||
"inputTokens": 4828,
|
||||
"outputTokens": 178,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "e142056d-56ab-4352-b091-b56054bd1359",
|
||||
"question": "Bob was invited to participate in a game show, and he advanced to the final round. The final round offered Bob the chance to win a large sum by playing a game against the host. The host has 30 shiny prop coins, each of which is worth $1,000 if Bob manages to win them by playing the game. The host hides the coins in three different prize boxes and then shuffles their order. The only rule restricting the host's coin placement is that one box must contain at least 2 coins, and one box must contain 6 more coins than another box. In order to play, Bob must submit three guesses, one guess for the number of coins in each box. The box is then opened and the number of coins is revealed. If Bob's guess is a number greater than the number of coins in the box, Bob earns no coins. If Bob guesses a number equal to or less than the number of coins in the box, Bob wins a number of coins equal to his guess.\n\nIf Bob plays uses the optimal strategy, what's the minimum amount of money he can win from the game?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "16000",
|
||||
"turns": 1,
|
||||
"wallMs": 25994,
|
||||
"inputTokens": 1456,
|
||||
"outputTokens": 1403,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "50ad0280-0819-4bd9-b275-5de32d3b5bcb",
|
||||
"question": "Pull out the sentence in the following 5x7 block of text. Read from left to right and use all of the letters in order:\n\nTHESE\nAGULL\nGLIDE\nDPEAC\nEFULL\nYTOMY\nCHAIR",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "THE SEAGULL GLIDED PEACEFULLY TO MY CHAIR",
|
||||
"expected_output": "The seagull glided peacefully to my chair.",
|
||||
"turns": 1,
|
||||
"wallMs": 3314,
|
||||
"inputTokens": 1276,
|
||||
"outputTokens": 139,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "7673d772-ef80-4f0f-a602-1bf4485c9b43",
|
||||
"question": "On Cornell Law School website's legal information institute, under the fifth section of federal rules alphabetically, what word was deleted in the last amendment to the first rule in the article that has \"witnesses\" in the most titles as of 2021?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "inference",
|
||||
"turns": 12,
|
||||
"wallMs": 105812,
|
||||
"inputTokens": 60236,
|
||||
"outputTokens": 1836,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "c365c1c7-a3db-4d5e-a9a1-66f56eae7865",
|
||||
"question": "Of the cities within the United States where U.S. presidents were born, which two are the farthest apart from the westernmost to the easternmost going east, giving the city names only? Give them to me in alphabetical order, in a comma-separated list",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "Honolulu, Quincy",
|
||||
"expected_output": "Braintree, Honolulu",
|
||||
"turns": 4,
|
||||
"wallMs": 36268,
|
||||
"inputTokens": 22752,
|
||||
"outputTokens": 956,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "7d4a7d1d-cac6-44a8-96e8-ea9584a70825",
|
||||
"question": "According to Girls Who Code, how long did it take in years for the percentage of computer scientists that were women to change by 13% from a starting point of 37%?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "33",
|
||||
"expected_output": "22",
|
||||
"turns": 10,
|
||||
"wallMs": 91860,
|
||||
"inputTokens": 63658,
|
||||
"outputTokens": 1157,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "dc22a632-937f-4e6a-b72f-ba0ff3f5ff97",
|
||||
"question": "What was the complete title of the book in which two James Beard Award winners recommended the restaurant where Ali Khan enjoyed a New Mexican staple in his cost-conscious TV show that started in 2015? Write the numbers in plain text if there are some in the title.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Five Hundred Things to Eat Before It's Too Late: And the Very Best Places to Eat Them",
|
||||
"expected_output": "Five Hundred Things To Eat Before It's Too Late: and the Very Best Places to Eat Them",
|
||||
"turns": 7,
|
||||
"wallMs": 56233,
|
||||
"inputTokens": 31394,
|
||||
"outputTokens": 1035,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "3f57289b-8c60-48be-bd80-01f8099ca449",
|
||||
"question": "How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "525",
|
||||
"expected_output": "519",
|
||||
"turns": 3,
|
||||
"wallMs": 19433,
|
||||
"inputTokens": 4798,
|
||||
"outputTokens": 222,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "23dd907f-1261-4488-b21c-e9185af91d5e",
|
||||
"question": "In Audre Lorde\u2019s poem \u201cFather Son and Holy Ghost\u201d, what is the number of the stanza in which some lines are indented?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "2",
|
||||
"expected_output": "2",
|
||||
"turns": 3,
|
||||
"wallMs": 21660,
|
||||
"inputTokens": 5030,
|
||||
"outputTokens": 456,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "1f975693-876d-457b-a649-393859e79bf3",
|
||||
"question": "Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(\n\nCould you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "132, 133, 134, 197, 245",
|
||||
"expected_output": "132, 133, 134, 197, 245",
|
||||
"turns": 2,
|
||||
"wallMs": 8467,
|
||||
"inputTokens": 3100,
|
||||
"outputTokens": 212,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "840bfca7-4f7b-481a-8794-c560c340185d",
|
||||
"question": "On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "80GSFC21M0002",
|
||||
"expected_output": "80GSFC21M0002",
|
||||
"turns": 5,
|
||||
"wallMs": 33926,
|
||||
"inputTokens": 14326,
|
||||
"outputTokens": 660,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "a0068077-79f4-461a-adfe-75c1a4148545",
|
||||
"question": "What was the actual enrollment count of the clinical trial on H. pylori in acne vulgaris patients from Jan-May 2018 as listed on the NIH website?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "90",
|
||||
"turns": 12,
|
||||
"wallMs": 128487,
|
||||
"inputTokens": 60672,
|
||||
"outputTokens": 1684,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "bda648d7-d618-4883-88f4-3466eabd860e",
|
||||
"question": "Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "Saint Petersburg",
|
||||
"expected_output": "Saint Petersburg",
|
||||
"turns": 2,
|
||||
"wallMs": 8364,
|
||||
"inputTokens": 3244,
|
||||
"outputTokens": 137,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "50ec8903-b81f-4257-9450-1085afd2c319",
|
||||
"question": "A standard Rubik\u2019s cube has been broken into cubes making up its sides. The cubes are jumbled, and one is removed. There are 6 cubes with one colored face, 12 edge cubes with two colored faces, and 8 corner cubes with three colored faces. All blue cubes have been found. All cubes directly left, right, above, and below the orange center cube have been found, along with the center cube. The green corners have all been found, along with all green that borders yellow. For all orange cubes found, the opposite face\u2019s cubes have been found. The removed cube has two colors on its faces. What are they? Answer using a comma separated list, with the colors ordered alphabetically.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "",
|
||||
"expected_output": "green, white",
|
||||
"turns": 2,
|
||||
"wallMs": 39410,
|
||||
"inputTokens": 4052,
|
||||
"outputTokens": 2140,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "cf106601-ab4f-4af9-b045-5295fe67b37d",
|
||||
"question": "What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "CUB",
|
||||
"expected_output": "CUB",
|
||||
"turns": 3,
|
||||
"wallMs": 16126,
|
||||
"inputTokens": 7604,
|
||||
"outputTokens": 254,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "a0c07678-e491-4bbc-8f0b-07405144218f",
|
||||
"question": "Who are the pitchers with the number before and after Taish\u014d Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "Yamasaki, Uehara",
|
||||
"expected_output": "Yoshida, Uehara",
|
||||
"turns": 4,
|
||||
"wallMs": 20212,
|
||||
"inputTokens": 8494,
|
||||
"outputTokens": 451,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "7bd855d8-463d-4ed5-93ca-5fe35145f733",
|
||||
"question": "The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": true,
|
||||
"answer": "$89706.00",
|
||||
"expected_output": "89706.00",
|
||||
"turns": 2,
|
||||
"wallMs": 9312,
|
||||
"inputTokens": 3480,
|
||||
"outputTokens": 603,
|
||||
"decomposed": false
|
||||
},
|
||||
{
|
||||
"task_id": "5a0c1adf-205e-4841-a666-7c3ef95def9d",
|
||||
"question": "What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"correct": false,
|
||||
"answer": "Claus Peter",
|
||||
"expected_output": "Claus",
|
||||
"turns": 2,
|
||||
"wallMs": 11071,
|
||||
"inputTokens": 3903,
|
||||
"outputTokens": 210,
|
||||
"decomposed": false
|
||||
}
|
||||
],
|
||||
"iter": 56,
|
||||
"mode": "codeagent",
|
||||
"harness": "ADR-138 CodeAgent (smolagents-style Python code blocks)",
|
||||
"branch": "feat/iter-54-codeagent-harness",
|
||||
"runDate": "2026-05-28T00:08:26-04:00",
|
||||
"completedDate": "2026-05-28T00:28:43.080293"
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"iter": 60,
|
||||
"branch": "feat/visit-webpage-tool",
|
||||
"pr": 2211,
|
||||
"model": "claude-sonnet-4-6",
|
||||
"mode": "hybrid",
|
||||
"total": 53,
|
||||
"pass": 28,
|
||||
"fail": 25,
|
||||
"pass_rate": 0.5283018867924528,
|
||||
"pass_rate_pct": "52.8%",
|
||||
"cost_usd": 6.1162,
|
||||
"mean_turns": 4.6,
|
||||
"mean_time_s": 35.8,
|
||||
"routing": {
|
||||
"ToolCalling": 46,
|
||||
"CodeAgent": 7
|
||||
},
|
||||
"routing_rules": {
|
||||
"web_retrieval": 15,
|
||||
"pure_reasoning": 7,
|
||||
"default": 16,
|
||||
"attachment": 11,
|
||||
"long_question": 4
|
||||
},
|
||||
"visit_webpage_invocations": 113,
|
||||
"empty_answer_failures": 19,
|
||||
"comparisons": {
|
||||
"iter56b_n1_high_draw": {
|
||||
"score": 35,
|
||||
"pct": "66.0%",
|
||||
"delta": -7
|
||||
},
|
||||
"iter56c_n1": {
|
||||
"score": 30,
|
||||
"pct": "56.6%",
|
||||
"delta": -2
|
||||
},
|
||||
"iter56d_n1": {
|
||||
"score": 31,
|
||||
"pct": "58.5%",
|
||||
"delta": -3
|
||||
},
|
||||
"n3_mean": {
|
||||
"score": 32.0,
|
||||
"pct": "60.4%",
|
||||
"delta": -4.0
|
||||
},
|
||||
"iter57_combined_naive": {
|
||||
"score": 25,
|
||||
"pct": "47.2%",
|
||||
"delta": "+3 vs iter57"
|
||||
}
|
||||
},
|
||||
"verdict": "within_n3_mean_variance",
|
||||
"autonomous_action": "keep_main_document",
|
||||
"notes": [
|
||||
"28/53 = 52.8% is above iter57 naive (25/53=47.2%) but below n3 mean (32.0/53=60.4%)",
|
||||
"19/25 failures are empty-answer, indicating FINAL_ANSWER extraction regression at some turns",
|
||||
"visit_webpage fired 113 times \u2014 tool is active and being invoked",
|
||||
"hybrid routing: 46 ToolCalling (87%), 7 CodeAgent (13%)",
|
||||
"Score falls in 30-32 neutral band per autonomous decision matrix",
|
||||
"Consistent with high variance single-run \u2014 not a clear regression vs main"
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -57,6 +57,14 @@ import {
|
||||
TextBlock,
|
||||
ContentBlock,
|
||||
} from './gaia-tools/index.js';
|
||||
import {
|
||||
checkConvergenceTriggers,
|
||||
createConvergenceState,
|
||||
forceCommit,
|
||||
recordTurn,
|
||||
argsHash as convergenceArgsHash,
|
||||
} from './gaia-convergence.js';
|
||||
import type { ConvergenceState } from './gaia-convergence.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
@@ -119,6 +127,10 @@ export interface GaiaAgentResult {
|
||||
/** Number of planning-checkpoint injections during this run (0 when planning is disabled). */
|
||||
replanCount?: number;
|
||||
timedOut?: boolean;
|
||||
/** Set when the convergence layer fired and committed the final answer. */
|
||||
convergenceTrigger?: string;
|
||||
/** True when the convergence layer recovered the answer from prior message history. */
|
||||
convergenceUsedFallback?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
@@ -146,6 +158,19 @@ export interface GaiaAgentOptions {
|
||||
* Exposed so callers can inject mocks for testing.
|
||||
*/
|
||||
catalogue?: GaiaToolCatalogue;
|
||||
/**
|
||||
* Enable the convergence layer (default: true).
|
||||
*
|
||||
* When enabled, the convergence layer monitors for three failure modes:
|
||||
* 1. max_turns hit without FINAL_ANSWER
|
||||
* 2. Loop (same tool+args 3× in a 5-turn window)
|
||||
* 3. Token overflow (>120k input tokens)
|
||||
*
|
||||
* On detection, a forced-commit phase is run: one API call with a
|
||||
* directive prompt, no tools, then a fallback scan of prior messages.
|
||||
* Set to false to disable (e.g. for ablation testing).
|
||||
*/
|
||||
enableConvergence?: boolean;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -485,6 +510,7 @@ export async function runGaiaAgent(
|
||||
planningInterval = PLANNING_INTERVAL,
|
||||
apiKey: suppliedKey,
|
||||
catalogue: suppliedCatalogue,
|
||||
enableConvergence = true,
|
||||
} = options;
|
||||
|
||||
const wallStart = Date.now();
|
||||
@@ -497,6 +523,9 @@ export async function runGaiaAgent(
|
||||
let totalOutputTokens = 0;
|
||||
let replanCount = 0;
|
||||
|
||||
// Convergence layer state — tracks turns, tokens, and tool call patterns.
|
||||
const convState: ConvergenceState = createConvergenceState();
|
||||
|
||||
const messages: MessageParam[] = [
|
||||
{ role: 'user', content: buildInitialContent(question) },
|
||||
];
|
||||
@@ -506,6 +535,48 @@ export async function runGaiaAgent(
|
||||
for (let turn = 0; turn < maxTurns; turn++) {
|
||||
turns = turn + 1;
|
||||
|
||||
// --- Convergence check: token overflow or loop (BEFORE the API call) ---
|
||||
if (enableConvergence) {
|
||||
const earlyTrigger = checkConvergenceTriggers(convState, maxTurns);
|
||||
if (earlyTrigger === 'token_overflow' || earlyTrigger === 'loop') {
|
||||
process.stderr.write(
|
||||
`[convergence] ${earlyTrigger} detected at turn ${turns} — forcing commit\n`,
|
||||
);
|
||||
const commitResult = await forceCommit(
|
||||
messages as Array<{ role: string; content: string | unknown }>,
|
||||
async (msgs) => {
|
||||
const r = await callAnthropicWithTools(
|
||||
apiKey, model,
|
||||
msgs as MessageParam[],
|
||||
[], // NO tools in forced-commit call
|
||||
maxTokensPerTurn,
|
||||
perTurnTimeoutMs,
|
||||
);
|
||||
const textParts = r.content
|
||||
.filter((b) => b.type === 'text')
|
||||
.map((b) => (b as TextBlock).text)
|
||||
.join('\n');
|
||||
totalInputTokens += r.usage.input_tokens;
|
||||
totalOutputTokens += r.usage.output_tokens;
|
||||
return textParts;
|
||||
},
|
||||
earlyTrigger,
|
||||
);
|
||||
return {
|
||||
questionId: question.task_id,
|
||||
finalAnswer: commitResult.answer,
|
||||
turns,
|
||||
toolCallsByName,
|
||||
totalInputTokens,
|
||||
totalOutputTokens,
|
||||
wallMs: Date.now() - wallStart,
|
||||
replanCount,
|
||||
convergenceTrigger: earlyTrigger,
|
||||
convergenceUsedFallback: commitResult.usedFallback,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let resp: AnthropicResponse;
|
||||
try {
|
||||
resp = await callAnthropicWithTools(
|
||||
@@ -533,6 +604,11 @@ export async function runGaiaAgent(
|
||||
totalInputTokens += resp.usage.input_tokens;
|
||||
totalOutputTokens += resp.usage.output_tokens;
|
||||
|
||||
// Update convergence state: record token usage for this turn (tool calls tracked below).
|
||||
if (enableConvergence) {
|
||||
recordTurn(convState, resp.usage.input_tokens, []);
|
||||
}
|
||||
|
||||
if (resp.stop_reason === 'end_turn' || resp.stop_reason === 'max_tokens') {
|
||||
const finalAnswer = extractFinalAnswer(resp);
|
||||
return {
|
||||
@@ -548,11 +624,29 @@ export async function runGaiaAgent(
|
||||
}
|
||||
|
||||
if (resp.stop_reason === 'tool_use') {
|
||||
// Track tool call counts before executing
|
||||
// Track tool call counts and update convergence state with this turn's tool calls.
|
||||
const toolCallsThisTurn: Array<{ name: string; args: object }> = [];
|
||||
for (const block of resp.content) {
|
||||
if (block.type === 'tool_use') {
|
||||
const toolBlock = block as ToolUseBlock;
|
||||
toolCallsByName[toolBlock.name] = (toolCallsByName[toolBlock.name] ?? 0) + 1;
|
||||
if (enableConvergence) {
|
||||
toolCallsThisTurn.push({
|
||||
name: toolBlock.name,
|
||||
args: (toolBlock.input ?? {}) as object,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Append tool call fingerprints to convergence state.
|
||||
if (enableConvergence && toolCallsThisTurn.length > 0) {
|
||||
for (const tc of toolCallsThisTurn) {
|
||||
convState.toolCalls.push({
|
||||
name: tc.name,
|
||||
argsHash: convergenceArgsHash(tc.name, tc.args),
|
||||
turn: turns,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -601,7 +695,46 @@ export async function runGaiaAgent(
|
||||
};
|
||||
}
|
||||
|
||||
// Exhausted maxTurns
|
||||
// Exhausted maxTurns — attempt convergence-layer forced commit if enabled.
|
||||
if (enableConvergence) {
|
||||
process.stderr.write(
|
||||
`[convergence] max_turns (${maxTurns}) exhausted — forcing commit\n`,
|
||||
);
|
||||
const commitResult = await forceCommit(
|
||||
messages as Array<{ role: string; content: string | unknown }>,
|
||||
async (msgs) => {
|
||||
const r = await callAnthropicWithTools(
|
||||
apiKey, model,
|
||||
msgs as MessageParam[],
|
||||
[], // NO tools in forced-commit call
|
||||
maxTokensPerTurn,
|
||||
perTurnTimeoutMs,
|
||||
);
|
||||
const textParts = r.content
|
||||
.filter((b) => b.type === 'text')
|
||||
.map((b) => (b as TextBlock).text)
|
||||
.join('\n');
|
||||
totalInputTokens += r.usage.input_tokens;
|
||||
totalOutputTokens += r.usage.output_tokens;
|
||||
return textParts;
|
||||
},
|
||||
'max_turns',
|
||||
);
|
||||
return {
|
||||
questionId: question.task_id,
|
||||
finalAnswer: commitResult.answer,
|
||||
turns,
|
||||
toolCallsByName,
|
||||
totalInputTokens,
|
||||
totalOutputTokens,
|
||||
wallMs: Date.now() - wallStart,
|
||||
replanCount,
|
||||
timedOut: !commitResult.answer,
|
||||
convergenceTrigger: 'max_turns',
|
||||
convergenceUsedFallback: commitResult.usedFallback,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
questionId: question.task_id,
|
||||
finalAnswer: null,
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
/**
|
||||
* Smoke tests for the GAIA convergence layer (gaia-convergence.ts).
|
||||
*
|
||||
* 10 assertions covering:
|
||||
* T1: max_turns hit → checkConvergenceTriggers returns 'max_turns'
|
||||
* T2: same tool+args 3× in window → returns 'loop'
|
||||
* T3: 120k tokens → returns 'token_overflow'
|
||||
* T4: forceCommit with prose-embedded answer in history → extraction succeeds
|
||||
* T5: forceCommit with explicit FINAL_ANSWER: X in response → returns X
|
||||
* T6: forceCommit on truly empty/looping conversation → returns null (graceful)
|
||||
* T7: argsHash deterministic — same args → same hash
|
||||
* T8: argsHash distinguishes — different args → different hash
|
||||
* T9: ConvergenceState records turn counts correctly via recordTurn
|
||||
* T10: Anti-loop window correctly slides — only last 5 turns matter
|
||||
*
|
||||
* Refs: #2156, iter 62
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
import {
|
||||
checkConvergenceTriggers,
|
||||
forceCommit,
|
||||
argsHash,
|
||||
createConvergenceState,
|
||||
recordTurn,
|
||||
extractFinalAnswerFromText,
|
||||
extractFromPriorMessages,
|
||||
TOKEN_OVERFLOW_THRESHOLD,
|
||||
LOOP_REPEAT_THRESHOLD,
|
||||
LOOP_WINDOW_SIZE,
|
||||
} from './gaia-convergence.js';
|
||||
import type { ConvergenceState } from './gaia-convergence.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test harness
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name: string, fn: () => void | Promise<void>): void {
|
||||
const result = fn();
|
||||
if (result && typeof (result as Promise<void>).then === 'function') {
|
||||
(result as Promise<void>)
|
||||
.then(() => {
|
||||
console.log(` PASS ${name}`);
|
||||
passed++;
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
console.error(` FAIL ${name}: ${(err as Error).message ?? err}`);
|
||||
failed++;
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
console.log(` PASS ${name}`);
|
||||
passed++;
|
||||
} catch (err: unknown) {
|
||||
console.error(` FAIL ${name}: ${(err as Error).message ?? err}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function testAsync(name: string, fn: () => Promise<void>): Promise<void> {
|
||||
try {
|
||||
await fn();
|
||||
console.log(` PASS ${name}`);
|
||||
passed++;
|
||||
} catch (err: unknown) {
|
||||
console.error(` FAIL ${name}: ${(err as Error).message ?? err}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T1: max_turns hit → returns 'max_turns'
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T1: max_turns hit returns max_turns', () => {
|
||||
const state: ConvergenceState = {
|
||||
turnCount: 12,
|
||||
totalTokens: 1000,
|
||||
toolCalls: [],
|
||||
detectedFailureMode: null,
|
||||
};
|
||||
const result = checkConvergenceTriggers(state, 12);
|
||||
assert.strictEqual(result, 'max_turns', `Expected 'max_turns', got ${result!}`);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T2: same tool+args 3× in last 5 turns → returns 'loop'
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T2: loop detection — same tool+args 3x in window', () => {
|
||||
const hash = argsHash('web_search', { query: 'test' });
|
||||
const state: ConvergenceState = {
|
||||
turnCount: 5,
|
||||
totalTokens: 5000,
|
||||
toolCalls: [
|
||||
{ name: 'web_search', argsHash: hash, turn: 1 },
|
||||
{ name: 'web_search', argsHash: hash, turn: 2 },
|
||||
{ name: 'web_search', argsHash: hash, turn: 3 },
|
||||
],
|
||||
detectedFailureMode: null,
|
||||
};
|
||||
const result = checkConvergenceTriggers(state, 15);
|
||||
assert.strictEqual(result, 'loop', `Expected 'loop', got ${result!}`);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T3: 120k tokens → returns 'token_overflow'
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T3: token_overflow at threshold', () => {
|
||||
const state: ConvergenceState = {
|
||||
turnCount: 5,
|
||||
totalTokens: TOKEN_OVERFLOW_THRESHOLD,
|
||||
toolCalls: [],
|
||||
detectedFailureMode: null,
|
||||
};
|
||||
const result = checkConvergenceTriggers(state, 20);
|
||||
assert.strictEqual(result, 'token_overflow', `Expected 'token_overflow', got ${result!}`);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T4: forceCommit with prose-embedded answer in prior history → extraction succeeds
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
await testAsync('T4: forceCommit — prose-embedded FINAL_ANSWER in prior history', async () => {
|
||||
const messages = [
|
||||
{ role: 'user', content: 'What is 2+2?' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'Let me think about this. The answer is definitely 4. FINAL_ANSWER: 4',
|
||||
},
|
||||
{ role: 'user', content: 'tool results...' },
|
||||
{ role: 'assistant', content: 'I got confused. Let me recalculate.' },
|
||||
];
|
||||
|
||||
// callModel returns an empty response (simulating no new FINAL_ANSWER)
|
||||
const callModel = async () => 'I need to think more about this.';
|
||||
|
||||
const result = await forceCommit(messages as Array<{ role: string; content: string | unknown }>, callModel, 'max_turns');
|
||||
|
||||
assert.strictEqual(result.answer, '4', `Expected '4', got ${result.answer!}`);
|
||||
assert.strictEqual(result.usedFallback, true, 'Expected usedFallback=true for history scan');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T5: forceCommit with explicit FINAL_ANSWER: X in forced response → returns X
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
await testAsync('T5: forceCommit — explicit FINAL_ANSWER in forced-commit response', async () => {
|
||||
const messages = [
|
||||
{ role: 'user', content: 'What is the capital of France?' },
|
||||
{ role: 'assistant', content: 'I need to search for this.' },
|
||||
];
|
||||
|
||||
// callModel returns a response WITH FINAL_ANSWER
|
||||
const callModel = async () => 'Based on my knowledge, the capital is Paris. FINAL_ANSWER: Paris';
|
||||
|
||||
const result = await forceCommit(messages as Array<{ role: string; content: string | unknown }>, callModel, 'max_turns');
|
||||
|
||||
assert.strictEqual(result.answer, 'Paris', `Expected 'Paris', got ${result.answer!}`);
|
||||
assert.strictEqual(result.usedFallback, false, 'Expected usedFallback=false when direct extraction');
|
||||
assert.strictEqual(result.triggerMode, 'max_turns');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T6: forceCommit on truly empty conversation → returns null (graceful)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
await testAsync('T6: forceCommit — graceful null on empty conversation', async () => {
|
||||
const messages = [
|
||||
{ role: 'user', content: 'What is something unknowable?' },
|
||||
{ role: 'assistant', content: 'I have been calling tools repeatedly with no progress.' },
|
||||
];
|
||||
|
||||
// callModel returns nothing useful
|
||||
const callModel = async () => 'I cannot determine the answer to this question.';
|
||||
|
||||
const result = await forceCommit(messages as Array<{ role: string; content: string | unknown }>, callModel, 'loop');
|
||||
|
||||
assert.strictEqual(result.answer, null, `Expected null, got ${result.answer!}`);
|
||||
assert.strictEqual(result.triggerMode, 'loop');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T7: argsHash deterministic — same args → same hash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T7: argsHash is deterministic', () => {
|
||||
const h1 = argsHash('web_search', { query: 'hello world', limit: 5 });
|
||||
const h2 = argsHash('web_search', { query: 'hello world', limit: 5 });
|
||||
assert.strictEqual(h1, h2, 'Same inputs must produce identical hash');
|
||||
assert.ok(h1.length === 16, `Hash should be 16 hex chars, got ${h1.length}`);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T8: argsHash distinguishes — different args → different hash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T8: argsHash distinguishes different args', () => {
|
||||
const h1 = argsHash('web_search', { query: 'hello' });
|
||||
const h2 = argsHash('web_search', { query: 'world' });
|
||||
assert.notStrictEqual(h1, h2, 'Different args must produce different hash');
|
||||
|
||||
const h3 = argsHash('calculator', { query: 'hello' });
|
||||
assert.notStrictEqual(h1, h3, 'Different tool names must produce different hash');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T9: ConvergenceState records turn counts correctly via recordTurn
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T9: recordTurn increments turnCount and totalTokens', () => {
|
||||
const state = createConvergenceState();
|
||||
|
||||
assert.strictEqual(state.turnCount, 0);
|
||||
assert.strictEqual(state.totalTokens, 0);
|
||||
assert.strictEqual(state.toolCalls.length, 0);
|
||||
|
||||
recordTurn(state, 1500, [
|
||||
{ name: 'web_search', args: { query: 'test' } },
|
||||
{ name: 'calculator', args: { expr: '2+2' } },
|
||||
]);
|
||||
|
||||
assert.strictEqual(state.turnCount, 1, `Expected turnCount=1, got ${state.turnCount}`);
|
||||
assert.strictEqual(state.totalTokens, 1500, `Expected totalTokens=1500, got ${state.totalTokens}`);
|
||||
assert.strictEqual(state.toolCalls.length, 2, `Expected 2 tool calls recorded, got ${state.toolCalls.length}`);
|
||||
assert.strictEqual(state.toolCalls[0].name, 'web_search');
|
||||
assert.strictEqual(state.toolCalls[0].turn, 1);
|
||||
|
||||
recordTurn(state, 2000, []);
|
||||
|
||||
assert.strictEqual(state.turnCount, 2);
|
||||
assert.strictEqual(state.totalTokens, 3500);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// T10: Anti-loop window correctly slides — only last LOOP_WINDOW_SIZE turns matter
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T10: loop window slides — old calls outside window do not trigger', () => {
|
||||
const hash = argsHash('web_search', { query: 'test' });
|
||||
|
||||
// 4 repeated calls followed by 5 different calls — window is last 5 only
|
||||
const oldCalls = Array.from({ length: 4 }, (_, i) => ({
|
||||
name: 'web_search', argsHash: hash, turn: i + 1,
|
||||
}));
|
||||
const recentCalls = Array.from({ length: LOOP_WINDOW_SIZE }, (_, i) => ({
|
||||
name: `tool_${i}`, argsHash: argsHash(`tool_${i}`, {}), turn: i + 5,
|
||||
}));
|
||||
|
||||
const state: ConvergenceState = {
|
||||
turnCount: 9,
|
||||
totalTokens: 9000,
|
||||
toolCalls: [...oldCalls, ...recentCalls],
|
||||
detectedFailureMode: null,
|
||||
};
|
||||
|
||||
const result = checkConvergenceTriggers(state, 20);
|
||||
assert.strictEqual(result, null,
|
||||
`Expected null (old calls outside window), got ${result!}`);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bonus: extractFinalAnswerFromText edge cases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('T11 (bonus): extractFinalAnswerFromText handles multiline and trailing content', () => {
|
||||
const text = 'After much deliberation:\nFINAL_ANSWER: 42\nSome extra text';
|
||||
const result = extractFinalAnswerFromText(text);
|
||||
assert.strictEqual(result, '42', `Expected '42', got ${result!}`);
|
||||
});
|
||||
|
||||
test('T12 (bonus): extractFromPriorMessages scans in reverse order', () => {
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Question' },
|
||||
{ role: 'assistant', content: 'First answer FINAL_ANSWER: wrong' },
|
||||
{ role: 'user', content: 'tool results' },
|
||||
{ role: 'assistant', content: 'After more research FINAL_ANSWER: correct' },
|
||||
];
|
||||
// Should return 'correct' (last assistant message first in reverse scan)
|
||||
const result = extractFromPriorMessages(messages as Array<{ role: string; content: string | unknown }>);
|
||||
assert.strictEqual(result, 'correct', `Expected 'correct' (most recent), got ${result!}`);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Summary
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Wait for all async tests to settle, then print summary
|
||||
setTimeout(() => {
|
||||
const total = passed + failed;
|
||||
console.log('');
|
||||
console.log('=== GAIA Convergence Layer Smoke Test ===');
|
||||
console.log(`Pass rate: ${passed}/${total}`);
|
||||
console.log(`Status: ${failed === 0 ? 'ALL PASSED' : `${failed} FAILED`}`);
|
||||
if (failed > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
}, 200);
|
||||
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* GAIA Convergence Layer — deterministic finalization for saturated agent loops.
|
||||
*
|
||||
* Detects three failure modes that cause empty FINAL_ANSWER extraction:
|
||||
* 1. max_turns hit without a final_answer call
|
||||
* 2. Loop detected (same tool + same args called 3× in a 5-turn window)
|
||||
* 3. Token budget exceeded (>120k tokens of conversation context)
|
||||
*
|
||||
* On detection, runs a FORCED COMMIT phase:
|
||||
* - Injects a stripped-down summary prompt: "Based on all observations, answer with
|
||||
* FINAL_ANSWER: X. Do NOT explore further."
|
||||
* - Makes 1 final API call with strict instruction and no tools.
|
||||
* - If still no FINAL_ANSWER in the response, runs the Stage 1 extraction cascade
|
||||
* against ALL prior assistant messages (last to first), returns first non-empty hit.
|
||||
* - If still empty: returns null but logs the failure mode.
|
||||
*
|
||||
* This is NOT a new cognition layer — it is a convergence controller.
|
||||
* It STRIPS information rather than adding it: one final chance to commit,
|
||||
* no tools available, prior context summarized rather than appended.
|
||||
*
|
||||
* Architecture principle (iter 60 post-mortem):
|
||||
* "More information can reduce agent reliability. Past a certain point —
|
||||
* retrieval depth, context size, browsing breadth, tool diversity — information
|
||||
* increases trajectory entropy. The system becomes less likely to finalize coherently."
|
||||
* This layer is the entropy-reducer.
|
||||
*
|
||||
* Refs: #2156, iter 60 (19/25 empty FINAL_ANSWER failures), iter 62
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Token threshold (sum of input tokens across all turns) that triggers overflow detection. */
|
||||
export const TOKEN_OVERFLOW_THRESHOLD = 120_000;
|
||||
|
||||
/** Number of repeated identical tool+args calls in a window that signals a loop. */
|
||||
export const LOOP_REPEAT_THRESHOLD = 3;
|
||||
|
||||
/** Sliding window size (turns) for loop detection. */
|
||||
export const LOOP_WINDOW_SIZE = 5;
|
||||
|
||||
/** Pattern Claude must output to signal it has a final answer. */
|
||||
const FINAL_ANSWER_RE = /FINAL_ANSWER:\s*(.+)/i;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Tracks state that the convergence layer needs to evaluate triggers. */
|
||||
export interface ConvergenceState {
|
||||
/** How many agent turns have elapsed so far. */
|
||||
turnCount: number;
|
||||
/** Sum of input tokens across all turns (used for overflow detection). */
|
||||
totalTokens: number;
|
||||
/** Ordered log of tool calls for loop detection. */
|
||||
toolCalls: Array<{ name: string; argsHash: string; turn: number }>;
|
||||
/** Set by checkConvergenceTriggers when a failure mode is detected. */
|
||||
detectedFailureMode: 'max_turns' | 'loop' | 'token_overflow' | null;
|
||||
}
|
||||
|
||||
/** Result of a forced-commit attempt. */
|
||||
export interface ForceCommitResult {
|
||||
/** The extracted answer, or null if even forced commit could not extract one. */
|
||||
answer: string | null;
|
||||
/** True when the answer was recovered from prior message history rather than
|
||||
* from the forced-commit API call. */
|
||||
usedFallback: boolean;
|
||||
/** The failure mode that triggered this forced commit. */
|
||||
triggerMode: ConvergenceState['detectedFailureMode'];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// argsHash — deterministic fingerprint of a tool call
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Produce a stable hash of a tool call's name + args for loop detection.
|
||||
*
|
||||
* Uses SHA-256 truncated to 16 hex chars — collision probability is
|
||||
* negligible for the small call volumes in a single agent run.
|
||||
*
|
||||
* The hash is deterministic: same toolName + same args always → same hash.
|
||||
* Different args always → different hash (within SHA-256 collision bounds).
|
||||
*/
|
||||
export function argsHash(toolName: string, args: object): string {
|
||||
const payload = toolName + '::' + JSON.stringify(args);
|
||||
return crypto.createHash('sha256').update(payload).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// checkConvergenceTriggers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Evaluate the current ConvergenceState and return the first failure mode
|
||||
* detected, or null if no trigger has fired.
|
||||
*
|
||||
* Evaluation order:
|
||||
* 1. max_turns — turnCount >= maxTurns
|
||||
* 2. token_overflow — totalTokens >= TOKEN_OVERFLOW_THRESHOLD
|
||||
* 3. loop — same tool+argsHash appears >= LOOP_REPEAT_THRESHOLD times
|
||||
* in the last LOOP_WINDOW_SIZE entries of toolCalls
|
||||
*
|
||||
* Only the FIRST matching trigger is returned (stops at first detection).
|
||||
* The caller is responsible for setting state.detectedFailureMode.
|
||||
*/
|
||||
export function checkConvergenceTriggers(
|
||||
state: ConvergenceState,
|
||||
maxTurns: number,
|
||||
): ConvergenceState['detectedFailureMode'] {
|
||||
// 1. max_turns
|
||||
if (state.turnCount >= maxTurns) {
|
||||
return 'max_turns';
|
||||
}
|
||||
|
||||
// 2. token_overflow
|
||||
if (state.totalTokens >= TOKEN_OVERFLOW_THRESHOLD) {
|
||||
return 'token_overflow';
|
||||
}
|
||||
|
||||
// 3. loop — check the sliding window
|
||||
const window = state.toolCalls.slice(-LOOP_WINDOW_SIZE);
|
||||
if (window.length >= LOOP_REPEAT_THRESHOLD) {
|
||||
const callKey = (c: { name: string; argsHash: string }) => c.name + '::' + c.argsHash;
|
||||
const counts = new Map<string, number>();
|
||||
for (const call of window) {
|
||||
const key = callKey(call);
|
||||
const count = (counts.get(key) ?? 0) + 1;
|
||||
counts.set(key, count);
|
||||
if (count >= LOOP_REPEAT_THRESHOLD) {
|
||||
return 'loop';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// buildForcedCommitPrompt
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build the stripped-down "commit now" prompt injected as a final user turn.
|
||||
*
|
||||
* Deliberately short and directive — this is an entropy-reducer, not a
|
||||
* context-enricher. Adding more information at this point would worsen the
|
||||
* problem we are solving.
|
||||
*/
|
||||
function buildForcedCommitPrompt(triggerMode: string): string {
|
||||
return (
|
||||
`[CONVERGENCE: ${triggerMode.toUpperCase()}]\n` +
|
||||
`You must commit to a final answer NOW. Do NOT call any more tools.\n` +
|
||||
`Based only on the information you have already gathered:\n` +
|
||||
`- Summarize your best answer in one line.\n` +
|
||||
`- Output it in EXACTLY this format: FINAL_ANSWER: <your answer>\n` +
|
||||
`- If you are uncertain, give your best estimate — do not say "I don't know".\n` +
|
||||
`You have ONE response. No further tool calls are permitted.`
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// extractFinalAnswerFromText — Stage 1 extraction against raw text
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Run Stage 1 extraction (FINAL_ANSWER: pattern) against a raw text string.
|
||||
* Returns the matched answer or null.
|
||||
*/
|
||||
export function extractFinalAnswerFromText(text: string): string | null {
|
||||
const match = FINAL_ANSWER_RE.exec(text);
|
||||
if (match && match[1] && match[1].trim()) {
|
||||
return match[1].trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// extractFromPriorMessages — fallback scan of message history
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Scan prior assistant messages from last to first, looking for FINAL_ANSWER.
|
||||
*
|
||||
* This is the fallback path when the forced-commit API call still does not
|
||||
* produce a FINAL_ANSWER. We search backwards because the most recent
|
||||
* assistant message is most likely to contain the best answer.
|
||||
*/
|
||||
export function extractFromPriorMessages(
|
||||
messages: Array<{ role: string; content: string | unknown }>,
|
||||
): string | null {
|
||||
// Iterate in reverse, assistant messages only
|
||||
const assistantMessages = messages
|
||||
.filter((m) => m.role === 'assistant')
|
||||
.reverse();
|
||||
|
||||
for (const msg of assistantMessages) {
|
||||
const text = typeof msg.content === 'string'
|
||||
? msg.content
|
||||
: extractTextFromContent(msg.content);
|
||||
|
||||
if (!text) continue;
|
||||
|
||||
const answer = extractFinalAnswerFromText(text);
|
||||
if (answer) return answer;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract plain text from a content block array (Anthropic API format).
|
||||
* Returns concatenated text from all text-type blocks, or empty string.
|
||||
*/
|
||||
function extractTextFromContent(content: unknown): string {
|
||||
if (!Array.isArray(content)) return '';
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (block && typeof block === 'object' && (block as { type: string }).type === 'text') {
|
||||
const text = (block as { type: string; text: string }).text;
|
||||
if (text) parts.push(text);
|
||||
}
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// forceCommit
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Run the forced-commit phase for a saturated agent loop.
|
||||
*
|
||||
* Steps:
|
||||
* 1. Append a stripped directive prompt to the message history.
|
||||
* 2. Call the model once with NO tools — forces text-only response.
|
||||
* 3. If FINAL_ANSWER found in the response → return it.
|
||||
* 4. If not → scan prior assistant messages last-to-first for FINAL_ANSWER.
|
||||
* 5. If still empty → return { answer: null, usedFallback: false }.
|
||||
*
|
||||
* The callModel callback must return the raw text response (assistant turn
|
||||
* text blocks concatenated). It should be called WITHOUT tool definitions
|
||||
* so the model cannot call tools in this turn.
|
||||
*
|
||||
* @param messages Current conversation history (mutated: 1 user turn appended)
|
||||
* @param callModel Callback that makes a single API call and returns the response text
|
||||
* @param triggerMode The failure mode that triggered this forced commit
|
||||
*/
|
||||
export async function forceCommit(
|
||||
messages: Array<{ role: string; content: string | unknown }>,
|
||||
callModel: (messages: Array<{ role: string; content: string | unknown }>) => Promise<string>,
|
||||
triggerMode: ConvergenceState['detectedFailureMode'] = 'max_turns',
|
||||
): Promise<ForceCommitResult> {
|
||||
const trigger = triggerMode ?? 'max_turns';
|
||||
|
||||
// Step 1: append the forced-commit directive
|
||||
const commitMessages = [
|
||||
...messages,
|
||||
{ role: 'user', content: buildForcedCommitPrompt(trigger) },
|
||||
];
|
||||
|
||||
// Step 2: call the model once, no tools
|
||||
let responseText = '';
|
||||
try {
|
||||
responseText = await callModel(commitMessages);
|
||||
} catch {
|
||||
// If the API call itself fails, fall through to fallback scan
|
||||
responseText = '';
|
||||
}
|
||||
|
||||
// Step 3: try to extract from the forced-commit response
|
||||
if (responseText) {
|
||||
const answer = extractFinalAnswerFromText(responseText);
|
||||
if (answer) {
|
||||
return { answer, usedFallback: false, triggerMode: triggerMode };
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: scan prior messages last-to-first
|
||||
const fallbackAnswer = extractFromPriorMessages(messages);
|
||||
if (fallbackAnswer) {
|
||||
return { answer: fallbackAnswer, usedFallback: true, triggerMode: triggerMode };
|
||||
}
|
||||
|
||||
// Step 5: graceful failure
|
||||
return { answer: null, usedFallback: false, triggerMode: triggerMode };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// createConvergenceState — factory for a fresh state object
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Create a new ConvergenceState with zero counters. */
|
||||
export function createConvergenceState(): ConvergenceState {
|
||||
return {
|
||||
turnCount: 0,
|
||||
totalTokens: 0,
|
||||
toolCalls: [],
|
||||
detectedFailureMode: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// recordTurn — update state after a completed turn
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Update ConvergenceState after each agent turn completes.
|
||||
*
|
||||
* @param state Mutable state object (modified in place)
|
||||
* @param inputTokens Tokens consumed in this turn
|
||||
* @param toolCallsThisTurn Tool calls made in this turn (name + args)
|
||||
*/
|
||||
export function recordTurn(
|
||||
state: ConvergenceState,
|
||||
inputTokens: number,
|
||||
toolCallsThisTurn: Array<{ name: string; args: object }>,
|
||||
): void {
|
||||
state.turnCount += 1;
|
||||
state.totalTokens += inputTokens;
|
||||
|
||||
for (const tc of toolCallsThisTurn) {
|
||||
state.toolCalls.push({
|
||||
name: tc.name,
|
||||
argsHash: argsHash(tc.name, tc.args),
|
||||
turn: state.turnCount,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -204,6 +204,12 @@ const runCommand: Command = {
|
||||
description: 'ADR-135 Track B: inject a planning checkpoint every N tool_use turns (default: 4, set 0 to disable). Based on smolagents finding — prevents tunnel-vision on bad strategies.',
|
||||
default: '4',
|
||||
},
|
||||
{
|
||||
name: 'enable-convergence',
|
||||
type: 'boolean',
|
||||
description: 'iter 62: enable convergence layer — forces a final commit when max_turns, loop, or token_overflow is detected (default: true). Disabling is for ablation only.',
|
||||
default: 'true',
|
||||
},
|
||||
],
|
||||
examples: [
|
||||
{
|
||||
@@ -271,6 +277,12 @@ const runCommand: Command = {
|
||||
const enableDecompose = ctx.flags['decompose'] === true || ctx.flags['decompose'] === 'true';
|
||||
// ADR-135 Track B: planning interval (passed through to runGaiaAgent via agentOpts).
|
||||
const planningInterval = parseInt(String(ctx.flags['planningInterval'] ?? ctx.flags['planning-interval'] ?? '4'), 10);
|
||||
// iter 62: convergence layer — default ON, disable with --no-enable-convergence.
|
||||
// Note: boolean false is falsy, so we check for explicit false values only.
|
||||
const enableConvergence = !(
|
||||
ctx.flags['enableConvergence'] === false || ctx.flags['enableConvergence'] === 'false' ||
|
||||
ctx.flags['enable-convergence'] === false || ctx.flags['enable-convergence'] === 'false'
|
||||
);
|
||||
|
||||
// Dynamic imports to avoid loading at startup.
|
||||
// NOTE: gaia-*.ts sources are pre-compiled under dist/src/benchmarks/ only --
|
||||
@@ -446,12 +458,13 @@ const runCommand: Command = {
|
||||
// Critic is suppressed when voting is active (same precedence rule as the global flag).
|
||||
const useThisCritic = enableCritic && !useThisVoting && runGaiaAgentWithCritic;
|
||||
|
||||
// Shared agent options (Track B planning interval wired here).
|
||||
// Shared agent options (Track B planning interval + iter 62 convergence wired here).
|
||||
const agentOpts = {
|
||||
model: effectiveModel,
|
||||
maxTurns: effectiveMaxTurns,
|
||||
planningInterval,
|
||||
apiKey,
|
||||
enableConvergence,
|
||||
};
|
||||
|
||||
// ADR-135 Track E: decompose the question if enabled.
|
||||
|
||||
Reference in new issue
Block a user