Deactivated o1 and misc improvements

.github/workflows/run-bot-aib-tournament.yaml

@@ -898,16 +898,16 @@ jobs:
   #     INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }}
   #     INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
 
-  bot_o1:
-    uses: ./.github/workflows/run-bot-launcher.yaml
-    with:
-      bot_name: 'METAC_O1_TOKEN'
-    secrets:
-      INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O1_TOKEN }}
-      INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }}
-      INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }}
-      INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+  # bot_o1:
+  #   uses: ./.github/workflows/run-bot-launcher.yaml
+  #   with:
+  #     bot_name: 'METAC_O1_TOKEN'
+  #   secrets:
+  #     INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O1_TOKEN }}
+  #     INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  #     INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }}
+  #     INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }}
+  #     INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
 
   # bot_o1_mini:
   #   uses: ./.github/workflows/run-bot-launcher.yaml

forecasting_tools/data_models/data_organizer.py

@@ -112,11 +112,11 @@ def get_report_type_for_question_type(
     def get_live_example_question_of_type(
         cls, question_type: type[MetaculusQuestion]
     ) -> MetaculusQuestion:
-        from forecasting_tools.helpers.metaculus_api import MetaculusApi
+        from forecasting_tools.helpers.metaculus_client import MetaculusClient
 
         assert issubclass(question_type, MetaculusQuestion)
         question_id = cls.get_example_post_id_for_question_type(question_type)
-        question = MetaculusApi.get_question_by_post_id(question_id)
+        question = MetaculusClient().get_question_by_post_id(question_id)
         assert isinstance(question, question_type)
         return question
 

forecasting_tools/data_models/questions.py

@@ -143,7 +143,7 @@ def from_metaculus_api_json(cls, post_api_json: dict) -> MetaculusQuestion:
         question = MetaculusQuestion(
             # NOTE: Reminder - When adding new fields, consider if group questions
             #       need to be parsed differently (i.e. if the field information is part of the post_json)
-            #       Also, anything that filters on the question level needs a local filter added to MetaculusApi (since the site does not filter subquestions)
+            #       Also, anything that filters on the question level needs a local filter added to MetaculusClient (since the site does not filter subquestions)
             state=question_state,
             question_text=question_json["title"],
             id_of_post=post_id,

forecasting_tools/helpers/metaculus_client.py

@@ -130,12 +130,13 @@ class MetaculusClient:
 
     Q3_2025_MARKET_PULSE_ID = "market-pulse-25q3"
     Q4_2025_MARKET_PULSE_ID = "market-pulse-25q4"
+    Q1_2026_MARKET_PULSE_ID = "market-pulse-26q1"
 
     CURRENT_METACULUS_CUP_ID = METACULUS_CUP_FALL_2025_ID
     CURRENT_QUARTERLY_CUP_ID = CURRENT_METACULUS_CUP_ID  # Consider this parameter deprecated since quarterly cup is no longer active
     CURRENT_AI_COMPETITION_ID = AIB_SPRING_2026_ID
     CURRENT_MINIBENCH_ID = "minibench"
-    CURRENT_MARKET_PULSE_ID = Q4_2025_MARKET_PULSE_ID
+    CURRENT_MARKET_PULSE_ID = Q1_2026_MARKET_PULSE_ID
 
     TEST_QUESTION_URLS = [
         "https://www.metaculus.com/questions/578/human-extinction-by-2100/",  # Human Extinction - Binary

run_bots.py

@@ -30,7 +30,8 @@
     UniformProbabilityBot,
 )
 from forecasting_tools.forecast_bots.template_bot import TemplateBot
-from forecasting_tools.helpers.metaculus_api import ApiFilter, MetaculusApi
+from forecasting_tools.helpers.metaculus_api import ApiFilter
+from forecasting_tools.helpers.metaculus_client import MetaculusClient
 from forecasting_tools.helpers.structure_output import DEFAULT_STRUCTURE_OUTPUT_MODEL
 
 logger = logging.getLogger(__name__)
@@ -42,6 +43,7 @@
 default_for_using_summary = False
 default_num_forecasts_for_research_only_bot = 3
 structure_output_model = DEFAULT_STRUCTURE_OUTPUT_MODEL
+default_metaculus_client = MetaculusClient()
 
 
 class ScheduleConfig:
@@ -84,10 +86,10 @@ def is_afternoon_window(cls, time: datetime | None = None) -> bool:
 
 
 class AllowedTourn(Enum):
-    MINIBENCH = MetaculusApi.CURRENT_MINIBENCH_ID
-    MAIN_AIB = MetaculusApi.CURRENT_AI_COMPETITION_ID
+    MINIBENCH = MetaculusClient.CURRENT_MINIBENCH_ID
+    MAIN_AIB = MetaculusClient.CURRENT_AI_COMPETITION_ID
     MAIN_SITE = "main-site"
-    METACULUS_CUP = MetaculusApi.CURRENT_METACULUS_CUP_ID
+    METACULUS_CUP = MetaculusClient.CURRENT_METACULUS_CUP_ID
     GULF_BREEZE = 32810  # https://www.metaculus.com/tournament/GB/
     DEMOCRACY_THREAT_INDEX = (
         32829  # https://www.metaculus.com/index/us-democracy-threat/
@@ -214,7 +216,7 @@ async def get_questions_for_config(
 
 
 def _get_aib_questions(tournament: AllowedTourn) -> list[MetaculusQuestion]:
-    aib_questions = MetaculusApi.get_all_open_questions_from_tournament(
+    aib_questions = default_metaculus_client.get_all_open_questions_from_tournament(
         tournament.value
     )
     filtered_questions = []
@@ -229,8 +231,10 @@ def _get__every_x_days__questions(
 ) -> list[MetaculusQuestion]:
     tournament_questions: list[MetaculusQuestion] = []
     for tournament in tournaments:
-        tournament_questions += MetaculusApi.get_all_open_questions_from_tournament(
-            tournament.value
+        tournament_questions += (
+            default_metaculus_client.get_all_open_questions_from_tournament(
+                tournament.value
+            )
         )
 
     filtered_questions = []
@@ -257,7 +261,7 @@ async def _get_questions_for_main_site(
         target_months_from_now = pendulum.now(tz="UTC").add(
             days=31 * months_ahead_to_check
         )
-        site_questions += await MetaculusApi.get_questions_matching_filter(
+        site_questions += await default_metaculus_client.get_questions_matching_filter(
             ApiFilter(
                 is_in_main_feed=True,
                 allowed_statuses=["open"],
@@ -270,9 +274,11 @@ async def _get_questions_for_main_site(
 
     other_tourns = [t for t in main_site_tourns if t != AllowedTourn.MAIN_SITE]
     for tournament in other_tourns:
-        site_questions += MetaculusApi.get_all_open_questions_from_tournament(
-            tournament.value,
-            group_question_mode="unpack_subquestions",
+        site_questions += (
+            default_metaculus_client.get_all_open_questions_from_tournament(
+                tournament.value,
+                group_question_mode="unpack_subquestions",
+            )
         )
 
     filtered_questions = []
@@ -1329,7 +1335,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]:  # NOSONAR
                     reasoning_effort="medium",
                 ),
             ),
-            "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP],
+            "tournaments": TournConfig.NONE,
         },
         "METAC_O1_MINI_TOKEN": {
             "estimated_cost_per_question": roughly_gpt_4o_cost,

scripts/generate_bot_costs_csv.py

@@ -0,0 +1,104 @@
+"""
+Script to generate a CSV table of bot costs per tournament.
+
+For each bot, shows:
+- The estimated cost per question
+- The cost for each tournament (cost if bot runs on that tournament, 0 if not)
+"""
+
+import csv
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from run_bots import AllowedTourn, get_default_bot_dict
+
+TOURNAMENT_SLUG_MAP: dict[AllowedTourn, str] = {
+    AllowedTourn.MINIBENCH: "minibench",
+    AllowedTourn.MAIN_AIB: "main_aib",
+    AllowedTourn.MAIN_SITE: "main_site",
+    AllowedTourn.METACULUS_CUP: "metaculus_cup",
+    AllowedTourn.GULF_BREEZE: "gulf_breeze",
+    AllowedTourn.DEMOCRACY_THREAT_INDEX: "democracy_threat_index",
+}
+
+
+def generate_bot_costs_csv(output_path: str | None = None) -> list[dict]:
+    bot_dict = get_default_bot_dict()
+    all_tournaments = list(AllowedTourn)
+
+    rows: list[dict] = []
+    for bot_name, config in bot_dict.items():
+        cost_per_question = config.estimated_cost_per_question or 0
+        bot_tournaments = config.tournaments
+
+        row = {
+            "bot_name": bot_name,
+            "cost_per_question": cost_per_question,
+        }
+
+        for tournament in all_tournaments:
+            slug = TOURNAMENT_SLUG_MAP.get(tournament, tournament.name.lower())
+            if tournament in bot_tournaments:
+                row[slug] = cost_per_question
+            else:
+                row[slug] = 0
+
+        rows.append(row)
+
+    rows.sort(key=lambda x: (x["cost_per_question"], x["bot_name"]))
+
+    if output_path:
+        fieldnames = ["bot_name", "cost_per_question"] + [
+            TOURNAMENT_SLUG_MAP.get(t, t.name.lower()) for t in all_tournaments
+        ]
+        with open(output_path, "w", newline="") as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(rows)
+        print(f"CSV written to {output_path}")
+
+    return rows
+
+
+def print_table(rows: list[dict]) -> None:
+    if not rows:
+        print("No bots found")
+        return
+
+    headers = list(rows[0].keys())
+    col_widths = {h: max(len(h), max(len(str(r[h])) for r in rows)) for h in headers}
+
+    header_line = " | ".join(h.ljust(col_widths[h]) for h in headers)
+    print(header_line)
+    print("-" * len(header_line))
+
+    for row in rows:
+        row_line = " | ".join(str(row[h]).ljust(col_widths[h]) for h in headers)
+        print(row_line)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Generate bot costs CSV")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default=None,
+        help="Output CSV file path. If not provided, prints to stdout.",
+    )
+    parser.add_argument(
+        "--print-table",
+        action="store_true",
+        help="Print a formatted table to stdout (in addition to CSV if -o is provided)",
+    )
+
+    args = parser.parse_args()
+
+    rows = generate_bot_costs_csv(args.output)
+
+    if args.print_table or not args.output:
+        print_table(rows)

scripts/run_benchmarker.py

@@ -37,7 +37,7 @@ async def benchmark_forecast_bots() -> None:
     bots = get_chosen_q2_bots()
     additional_code_to_snapshot = []
     # num_questions_to_use = 150
-    # chosen_questions = MetaculusApi.get_benchmark_questions(
+    # chosen_questions = MetaculusClient().get_benchmark_questions(
     #     num_questions_to_use,
     # )
     snapshots = QuestionPlusResearch.load_json_from_file_path(

scripts/run_key_factors.py

@@ -10,7 +10,7 @@
     ScoredKeyFactor,
 )
 from forecasting_tools.data_models.questions import MetaculusQuestion, QuestionState
-from forecasting_tools.helpers.metaculus_api import MetaculusApi
+from forecasting_tools.helpers.metaculus_client import MetaculusClient
 from forecasting_tools.util import file_manipulation
 from forecasting_tools.util.custom_logger import CustomLogger
 
@@ -58,7 +58,7 @@ async def run_key_factors_on_tournament(
     if tournament_id is None:
         questions = post_id_or_string_question
     else:
-        tournament_questions = MetaculusApi.get_all_open_questions_from_tournament(
+        tournament_questions = MetaculusClient().get_all_open_questions_from_tournament(
             tournament_id,
         )
         open_questions = [
@@ -103,7 +103,7 @@ async def _process_question(question: int | str | MetaculusQuestion) -> dict:
             api_json={},
         )
     if isinstance(question, int):
-        metaculus_question = MetaculusApi.get_question_by_post_id(question)
+        metaculus_question = MetaculusClient().get_question_by_post_id(question)
     else:
         assert isinstance(question, MetaculusQuestion)
         metaculus_question = question

scripts/run_question_snapshots.py

@@ -11,7 +11,7 @@
 )
 from forecasting_tools.data_models.data_organizer import DataOrganizer
 from forecasting_tools.data_models.questions import MetaculusQuestion
-from forecasting_tools.helpers.metaculus_api import MetaculusApi
+from forecasting_tools.helpers.metaculus_client import MetaculusClient
 from forecasting_tools.util.custom_logger import CustomLogger
 
 logger = logging.getLogger(__name__)
@@ -21,7 +21,7 @@ async def grab_questions(include_research: bool) -> None:
     # --- Parameters ---
     target_questions_to_use = 500
     target_training_size = 50
-    chosen_questions = MetaculusApi.get_benchmark_questions(
+    chosen_questions = MetaculusClient().get_benchmark_questions(
         target_questions_to_use,
         max_days_since_opening=365 + 180,
         days_to_resolve_in=None,