From 8a00ff4e075b4de381320051835e318ea0637f4e Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 31 Dec 2025 00:33:28 +0000 Subject: [PATCH] Deactivated o1 --- .github/workflows/run-bot-aib-tournament.yaml | 20 ++-- .../data_models/data_organizer.py | 4 +- forecasting_tools/data_models/questions.py | 2 +- forecasting_tools/helpers/metaculus_client.py | 3 +- run_bots.py | 30 +++-- scripts/generate_bot_costs_csv.py | 104 ++++++++++++++++++ scripts/run_benchmarker.py | 2 +- scripts/run_key_factors.py | 6 +- scripts/run_question_snapshots.py | 4 +- 9 files changed, 143 insertions(+), 32 deletions(-) create mode 100644 scripts/generate_bot_costs_csv.py diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 35532395..2b5559c8 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -898,16 +898,16 @@ jobs: # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_o1: - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: 'METAC_O1_TOKEN' - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O1_TOKEN }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_o1: + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: 'METAC_O1_TOKEN' + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O1_TOKEN }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} # bot_o1_mini: # uses: ./.github/workflows/run-bot-launcher.yaml diff --git a/forecasting_tools/data_models/data_organizer.py b/forecasting_tools/data_models/data_organizer.py index dd260911..85d336fc 100644 --- a/forecasting_tools/data_models/data_organizer.py +++ b/forecasting_tools/data_models/data_organizer.py @@ -112,11 +112,11 @@ def get_report_type_for_question_type( def get_live_example_question_of_type( cls, question_type: type[MetaculusQuestion] ) -> MetaculusQuestion: - from forecasting_tools.helpers.metaculus_api import MetaculusApi + from forecasting_tools.helpers.metaculus_client import MetaculusClient assert issubclass(question_type, MetaculusQuestion) question_id = cls.get_example_post_id_for_question_type(question_type) - question = MetaculusApi.get_question_by_post_id(question_id) + question = MetaculusClient().get_question_by_post_id(question_id) assert isinstance(question, question_type) return question diff --git a/forecasting_tools/data_models/questions.py b/forecasting_tools/data_models/questions.py index ee37feed..daa1cf18 100644 --- a/forecasting_tools/data_models/questions.py +++ b/forecasting_tools/data_models/questions.py @@ -143,7 +143,7 @@ def from_metaculus_api_json(cls, post_api_json: dict) -> MetaculusQuestion: question = MetaculusQuestion( # NOTE: Reminder - When adding new fields, consider if group questions # need to be parsed differently (i.e. if the field information is part of the post_json) - # Also, anything that filters on the question level needs a local filter added to MetaculusApi (since the site does not filter subquestions) + # Also, anything that filters on the question level needs a local filter added to MetaculusClient (since the site does not filter subquestions) state=question_state, question_text=question_json["title"], id_of_post=post_id, diff --git a/forecasting_tools/helpers/metaculus_client.py b/forecasting_tools/helpers/metaculus_client.py index 594008c0..896f1ad9 100644 --- a/forecasting_tools/helpers/metaculus_client.py +++ b/forecasting_tools/helpers/metaculus_client.py @@ -130,12 +130,13 @@ class MetaculusClient: Q3_2025_MARKET_PULSE_ID = "market-pulse-25q3" Q4_2025_MARKET_PULSE_ID = "market-pulse-25q4" + Q1_2026_MARKET_PULSE_ID = "market-pulse-26q1" CURRENT_METACULUS_CUP_ID = METACULUS_CUP_FALL_2025_ID CURRENT_QUARTERLY_CUP_ID = CURRENT_METACULUS_CUP_ID # Consider this parameter deprecated since quarterly cup is no longer active CURRENT_AI_COMPETITION_ID = AIB_SPRING_2026_ID CURRENT_MINIBENCH_ID = "minibench" - CURRENT_MARKET_PULSE_ID = Q4_2025_MARKET_PULSE_ID + CURRENT_MARKET_PULSE_ID = Q1_2026_MARKET_PULSE_ID TEST_QUESTION_URLS = [ "https://www.metaculus.com/questions/578/human-extinction-by-2100/", # Human Extinction - Binary diff --git a/run_bots.py b/run_bots.py index 02bacda8..51a32338 100644 --- a/run_bots.py +++ b/run_bots.py @@ -30,7 +30,8 @@ UniformProbabilityBot, ) from forecasting_tools.forecast_bots.template_bot import TemplateBot -from forecasting_tools.helpers.metaculus_api import ApiFilter, MetaculusApi +from forecasting_tools.helpers.metaculus_api import ApiFilter +from forecasting_tools.helpers.metaculus_client import MetaculusClient from forecasting_tools.helpers.structure_output import DEFAULT_STRUCTURE_OUTPUT_MODEL logger = logging.getLogger(__name__) @@ -42,6 +43,7 @@ default_for_using_summary = False default_num_forecasts_for_research_only_bot = 3 structure_output_model = DEFAULT_STRUCTURE_OUTPUT_MODEL +default_metaculus_client = MetaculusClient() class ScheduleConfig: @@ -84,10 +86,10 @@ def is_afternoon_window(cls, time: datetime | None = None) -> bool: class AllowedTourn(Enum): - MINIBENCH = MetaculusApi.CURRENT_MINIBENCH_ID - MAIN_AIB = MetaculusApi.CURRENT_AI_COMPETITION_ID + MINIBENCH = MetaculusClient.CURRENT_MINIBENCH_ID + MAIN_AIB = MetaculusClient.CURRENT_AI_COMPETITION_ID MAIN_SITE = "main-site" - METACULUS_CUP = MetaculusApi.CURRENT_METACULUS_CUP_ID + METACULUS_CUP = MetaculusClient.CURRENT_METACULUS_CUP_ID GULF_BREEZE = 32810 # https://www.metaculus.com/tournament/GB/ DEMOCRACY_THREAT_INDEX = ( 32829 # https://www.metaculus.com/index/us-democracy-threat/ @@ -214,7 +216,7 @@ async def get_questions_for_config( def _get_aib_questions(tournament: AllowedTourn) -> list[MetaculusQuestion]: - aib_questions = MetaculusApi.get_all_open_questions_from_tournament( + aib_questions = default_metaculus_client.get_all_open_questions_from_tournament( tournament.value ) filtered_questions = [] @@ -229,8 +231,10 @@ def _get__every_x_days__questions( ) -> list[MetaculusQuestion]: tournament_questions: list[MetaculusQuestion] = [] for tournament in tournaments: - tournament_questions += MetaculusApi.get_all_open_questions_from_tournament( - tournament.value + tournament_questions += ( + default_metaculus_client.get_all_open_questions_from_tournament( + tournament.value + ) ) filtered_questions = [] @@ -257,7 +261,7 @@ async def _get_questions_for_main_site( target_months_from_now = pendulum.now(tz="UTC").add( days=31 * months_ahead_to_check ) - site_questions += await MetaculusApi.get_questions_matching_filter( + site_questions += await default_metaculus_client.get_questions_matching_filter( ApiFilter( is_in_main_feed=True, allowed_statuses=["open"], @@ -270,9 +274,11 @@ async def _get_questions_for_main_site( other_tourns = [t for t in main_site_tourns if t != AllowedTourn.MAIN_SITE] for tournament in other_tourns: - site_questions += MetaculusApi.get_all_open_questions_from_tournament( - tournament.value, - group_question_mode="unpack_subquestions", + site_questions += ( + default_metaculus_client.get_all_open_questions_from_tournament( + tournament.value, + group_question_mode="unpack_subquestions", + ) ) filtered_questions = [] @@ -1329,7 +1335,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR reasoning_effort="medium", ), ), - "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], + "tournaments": TournConfig.NONE, }, "METAC_O1_MINI_TOKEN": { "estimated_cost_per_question": roughly_gpt_4o_cost, diff --git a/scripts/generate_bot_costs_csv.py b/scripts/generate_bot_costs_csv.py new file mode 100644 index 00000000..6f61ce1a --- /dev/null +++ b/scripts/generate_bot_costs_csv.py @@ -0,0 +1,104 @@ +""" +Script to generate a CSV table of bot costs per tournament. + +For each bot, shows: +- The estimated cost per question +- The cost for each tournament (cost if bot runs on that tournament, 0 if not) +""" + +import csv +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from run_bots import AllowedTourn, get_default_bot_dict + +TOURNAMENT_SLUG_MAP: dict[AllowedTourn, str] = { + AllowedTourn.MINIBENCH: "minibench", + AllowedTourn.MAIN_AIB: "main_aib", + AllowedTourn.MAIN_SITE: "main_site", + AllowedTourn.METACULUS_CUP: "metaculus_cup", + AllowedTourn.GULF_BREEZE: "gulf_breeze", + AllowedTourn.DEMOCRACY_THREAT_INDEX: "democracy_threat_index", +} + + +def generate_bot_costs_csv(output_path: str | None = None) -> list[dict]: + bot_dict = get_default_bot_dict() + all_tournaments = list(AllowedTourn) + + rows: list[dict] = [] + for bot_name, config in bot_dict.items(): + cost_per_question = config.estimated_cost_per_question or 0 + bot_tournaments = config.tournaments + + row = { + "bot_name": bot_name, + "cost_per_question": cost_per_question, + } + + for tournament in all_tournaments: + slug = TOURNAMENT_SLUG_MAP.get(tournament, tournament.name.lower()) + if tournament in bot_tournaments: + row[slug] = cost_per_question + else: + row[slug] = 0 + + rows.append(row) + + rows.sort(key=lambda x: (x["cost_per_question"], x["bot_name"])) + + if output_path: + fieldnames = ["bot_name", "cost_per_question"] + [ + TOURNAMENT_SLUG_MAP.get(t, t.name.lower()) for t in all_tournaments + ] + with open(output_path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + print(f"CSV written to {output_path}") + + return rows + + +def print_table(rows: list[dict]) -> None: + if not rows: + print("No bots found") + return + + headers = list(rows[0].keys()) + col_widths = {h: max(len(h), max(len(str(r[h])) for r in rows)) for h in headers} + + header_line = " | ".join(h.ljust(col_widths[h]) for h in headers) + print(header_line) + print("-" * len(header_line)) + + for row in rows: + row_line = " | ".join(str(row[h]).ljust(col_widths[h]) for h in headers) + print(row_line) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Generate bot costs CSV") + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help="Output CSV file path. If not provided, prints to stdout.", + ) + parser.add_argument( + "--print-table", + action="store_true", + help="Print a formatted table to stdout (in addition to CSV if -o is provided)", + ) + + args = parser.parse_args() + + rows = generate_bot_costs_csv(args.output) + + if args.print_table or not args.output: + print_table(rows) diff --git a/scripts/run_benchmarker.py b/scripts/run_benchmarker.py index f6ebfae0..4027e853 100644 --- a/scripts/run_benchmarker.py +++ b/scripts/run_benchmarker.py @@ -37,7 +37,7 @@ async def benchmark_forecast_bots() -> None: bots = get_chosen_q2_bots() additional_code_to_snapshot = [] # num_questions_to_use = 150 - # chosen_questions = MetaculusApi.get_benchmark_questions( + # chosen_questions = MetaculusClient().get_benchmark_questions( # num_questions_to_use, # ) snapshots = QuestionPlusResearch.load_json_from_file_path( diff --git a/scripts/run_key_factors.py b/scripts/run_key_factors.py index 65269227..3ff4f4d2 100644 --- a/scripts/run_key_factors.py +++ b/scripts/run_key_factors.py @@ -10,7 +10,7 @@ ScoredKeyFactor, ) from forecasting_tools.data_models.questions import MetaculusQuestion, QuestionState -from forecasting_tools.helpers.metaculus_api import MetaculusApi +from forecasting_tools.helpers.metaculus_client import MetaculusClient from forecasting_tools.util import file_manipulation from forecasting_tools.util.custom_logger import CustomLogger @@ -58,7 +58,7 @@ async def run_key_factors_on_tournament( if tournament_id is None: questions = post_id_or_string_question else: - tournament_questions = MetaculusApi.get_all_open_questions_from_tournament( + tournament_questions = MetaculusClient().get_all_open_questions_from_tournament( tournament_id, ) open_questions = [ @@ -103,7 +103,7 @@ async def _process_question(question: int | str | MetaculusQuestion) -> dict: api_json={}, ) if isinstance(question, int): - metaculus_question = MetaculusApi.get_question_by_post_id(question) + metaculus_question = MetaculusClient().get_question_by_post_id(question) else: assert isinstance(question, MetaculusQuestion) metaculus_question = question diff --git a/scripts/run_question_snapshots.py b/scripts/run_question_snapshots.py index bb024b4e..07b73f75 100644 --- a/scripts/run_question_snapshots.py +++ b/scripts/run_question_snapshots.py @@ -11,7 +11,7 @@ ) from forecasting_tools.data_models.data_organizer import DataOrganizer from forecasting_tools.data_models.questions import MetaculusQuestion -from forecasting_tools.helpers.metaculus_api import MetaculusApi +from forecasting_tools.helpers.metaculus_client import MetaculusClient from forecasting_tools.util.custom_logger import CustomLogger logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ async def grab_questions(include_research: bool) -> None: # --- Parameters --- target_questions_to_use = 500 target_training_size = 50 - chosen_questions = MetaculusApi.get_benchmark_questions( + chosen_questions = MetaculusClient().get_benchmark_questions( target_questions_to_use, max_days_since_opening=365 + 180, days_to_resolve_in=None,