chore: update r1/o3-mini, fix buttons

app.py

@@ -45,7 +45,7 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
     ) -> tuple[pd.DataFrame, list[str], str, str]:
         always_here_cols = [c.pretty_name for c in columns_dict.values() if c.never_hidden]
         selected_columns = [
-            c.pretty_name for c in columns_dict.values() if current_tag in c.tags and c not in always_here_cols
+            c.pretty_name for c in columns_dict.values() if current_tag in c.tags and c.pretty_name not in always_here_cols
         ]
         # keep the order of the columns
         filtered_df = full_df[[c for c in full_df.columns if c in (always_here_cols + selected_columns)]]
@@ -66,7 +66,6 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
         if filter_name == "All":
             return full_df[current_df.columns]
         else:
-            # actually filter by emoji
             query_symbol = filter_name[0]
             filtered_df = full_df[full_df[columns_dict["model_type_symbol"].pretty_name] == query_symbol]
             return filtered_df[current_df.columns]
@@ -77,7 +76,6 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
         if filter_name == "All":
             return full_df[current_df.columns]
         else:
-            # actually filter by emoji
             query_symbol = filter_name[0]
             filtered_df = full_df[full_df[columns_dict["model_size_symbol"].pretty_name] == query_symbol]
             return filtered_df[current_df.columns]
@@ -92,62 +90,64 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
             return filtered_df[current_df.columns]
 
     with gr.Column() as col:
-        # Add the controls
-        with gr.Accordion("➡️ See All Columns", open=False):
-            columns_to_select_visibility = [
-                c.pretty_name for c in columns_dict.values() if not c.fully_hidden and not c.never_hidden
-            ]
-            all_columns_selector = gr.CheckboxGroup(
-                choices=columns_to_select_visibility,
-                value=[
-                    c.pretty_name
-                    for c in columns_dict.values()
-                    if c.pretty_name in columns_to_select_visibility and c.displayed_by_default
-                ],
-                label="Select Columns to Display:",
-                interactive=True,
-                container=False,
-            )
         with gr.Row():
-            with gr.Column():
-                search_bar = gr.Textbox(
-                    placeholder="🔍 Search for your model and press ENTER...",
-                    show_label=False,
-                    elem_id="search-bar",
-                )
+            with gr.Column(scale=8):
+                with gr.Accordion("➡️ See All Columns", open=False):
+                    columns_to_select_visibility = [
+                        c.pretty_name for c in columns_dict.values() if not c.fully_hidden and not c.never_hidden
+                    ]
+                    all_columns_selector = gr.CheckboxGroup(
+                        choices=columns_to_select_visibility,
+                        value=[
+                            c.pretty_name
+                            for c in columns_dict.values()
+                            if c.pretty_name in columns_to_select_visibility and c.displayed_by_default
+                        ],
+                        label="Select Columns to Display:",
+                        interactive=True,
+                        container=False,
+                    )
+                with gr.Column(variant='panel'):
+                    gr.Markdown("Visible Columns:", elem_id="visible-columns-label")
+                    all_tags = {}
+                    with gr.Row():
+                        for c in columns_dict.values():
+                            for tag in c.tags:
+                                if tag not in all_tags:
+                                    all_tags[tag] = gr.Button(tag, interactive=True, size="sm", variant="secondary", min_width=100)
+            with gr.Column(scale=8):
+                with gr.Row():
+                    search_bar = gr.Textbox(
+                        placeholder="🔍 Search for your model and press ENTER...",
+                        show_label=False,
+                        elem_id="search-bar",
+                    )
 
-            # collect all column tags and create buttons for them
-            all_tags = {}
-            with gr.Column(variant="panel"):
-                gr.Markdown("Select Columns:")
-                for c in columns_dict.values():
-                    for tag in c.tags:
-                        if tag not in all_tags:
-                            all_tags[tag] = gr.Button(tag, interactive=True, size="sm")
-
-            model_type_filter_selector = gr.Radio(
-                label="Filter model types:",
-                choices=["All", "💙 Open-Weights", "🟥 Proprietary"],
-                value="All",
-                elem_id="model-type-filter",
-                interactive=True,
-            )
-
-            model_size_filter_selector = gr.Radio(
-                label="Filter model sizes:",
-                choices=["All", "🛴 Tiny (<5B)", "🚗 Small (5-50B)", "🚚 Medium (50-100B)", "🚀 Large (>100B)"],
-                value="All",
-                elem_id="model-size-filter",
-                interactive=True,
-            )
-
-            model_family_filter_selector = gr.Radio(
-                label="Filter model families:",
-                choices=["All"] + list(dataframe[columns_dict["model_family"].pretty_name].unique()),
-                value="All",
-                elem_id="model-family-filter",
-                interactive=True,
-            )
+                with gr.Row():
+                    model_type_filter_selector = gr.Dropdown(
+                        label="Filter model types:",
+                        choices=["All", "💙 Open-Weights", "🟥 Proprietary"],
+                        value="All",
+                        elem_id="model-type-filter",
+                        interactive=True,
+                        multiselect=False,
+                    )
+                    model_size_filter_selector = gr.Dropdown(
+                        label="Filter model sizes:",
+                        choices=["All", "🛴 Tiny (<5B)", "🚗 Small (5-50B)", "🚚 Medium (50-100B)", "🚀 Large (>100B)"],
+                        value="All",
+                        elem_id="model-size-filter",
+                        interactive=True,
+                        multiselect=False,
+                    )
+                    model_family_filter_selector = gr.Dropdown(
+                        label="Filter model families:",
+                        choices=["All"] + list(dataframe[columns_dict["model_family"].pretty_name].unique()),
+                        value="All",
+                        elem_id="model-family-filter",
+                        interactive=True,
+                        multiselect=False,
+                    )
 
         # create the hidden and visible dataframes to display
         hidden_leaderboard_df = gr.components.Dataframe(
@@ -162,7 +162,8 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
             elem_id="leaderboard-df",
             interactive=False,
         )
-        # add the callbacks
+
+        # Add the callbacks
         all_columns_selector.change(
             fn=filter_dataframe_by_selected_columns,
             inputs=[hidden_leaderboard_df, all_columns_selector],
@@ -188,6 +189,7 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
             inputs=[hidden_leaderboard_df, leaderboard_df, model_family_filter_selector],
             outputs=[leaderboard_df],
         )
+        # Wire up each visible-column button to filter by tag
         for tag, button in all_tags.items():
             button.click(
                 fn=filter_dataframe_by_selected_tag_columns,
@@ -195,7 +197,7 @@ def init_leaderboard(dataframe: pd.DataFrame, columns_dict: dict[str, Field]) ->
                 outputs=[leaderboard_df, all_columns_selector, model_type_filter_selector, model_size_filter_selector],
             )
 
-        # reload the leaderboard on the first load
+        # On first load, show the default columns
         filter_dataframe_by_selected_columns(dataframe, all_columns_selector.value)
         return col
 

data/u_math_eval_results.json

@@ -1403,42 +1403,81 @@
       0.5
     ]
   },
-  {
+    {
     "model_name": "deepseek-ai/DeepSeek-R1",
     "judge_model_name": "gpt-4o-2024-08-06",
     "u_math": [
-      63.2727,
-      73.6667,
-      16.5
+      68.3636,
+      79.0,
+      20.5
     ],
     "algebra": [
-      0.8,
-      0.96,
+      0.8056,
+      0.9667,
       0.0
     ],
     "differential_calc": [
-      0.4818,
-      0.6667,
-      0.0857
+      0.5136,
+      0.7067,
+      0.1
     ],
     "integral_calc": [
-      0.3606,
-      0.4,
-      0.2586
+      0.4471,
+      0.48,
+      0.3621
     ],
     "multivariable_calculus": [
-      0.6124,
-      0.6667,
+      0.6966,
+      0.7667,
       0.3214
     ],
     "precalculus_review": [
-      0.9188,
-      0.9733,
+      0.925,
+      0.98,
       0.1
     ],
     "sequences_series": [
-      0.7468,
+      0.8377,
+      0.84,
+      0.75
+    ]
+  },
+  {
+    "model_name": "o3-mini",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      68.9091,
+      79.6667,
+      20.5
+    ],
+    "algebra": [
+      0.7944,
+      0.94,
+      0.0667
+    ],
+    "differential_calc": [
+      0.5455,
+      0.7667,
+      0.0714
+    ],
+    "integral_calc": [
+      0.4904,
+      0.5267,
+      0.3966
+    ],
+    "multivariable_calculus": [
+      0.6854,
       0.7533,
+      0.3214
+    ],
+    "precalculus_review": [
+      0.9,
+      0.96,
+      0.0
+    ],
+    "sequences_series": [
+      0.8247,
+      0.8333,
       0.5
     ]
   },

src/populate.py

@@ -11,19 +11,31 @@ UNKNOWN_MODEL_SHOW_SIZE = 150
 PERCENT_ROUND_DIGITS = 1
 
 
+MODEL_CONFIG_CACHE = {}
+MODEL_CARD_CACHE = {}
+
+
 def get_hf_model_info_card_or_none(model_name: str) -> ModelInfo | None:
+    if model_name in MODEL_CARD_CACHE:
+        return MODEL_CARD_CACHE[model_name]
     try:
         info = model_info(repo_id=model_name)
+        MODEL_CARD_CACHE[model_name] = info
         return info
     except Exception:
+        MODEL_CARD_CACHE[model_name] = None
         return None
 
 
 def get_hf_hub_config_or_none(model_name: str) -> AutoConfig | None:
+    if model_name in MODEL_CONFIG_CACHE:
+        return MODEL_CONFIG_CACHE[model_name]
     try:
         config = AutoConfig.from_pretrained(model_name, revision="main", trust_remote_code=True)
+        MODEL_CONFIG_CACHE[model_name] = config
         return config
     except Exception:
+        MODEL_CONFIG_CACHE[model_name] = None
         return None
 
 
@@ -169,9 +181,9 @@ U_MATH_COLUMNS_DICT = {
     "rank": Field("Rank", "number", never_hidden=True),
     **MODEL_COLUMNS_DICT,
     "judge_model_name": Field("Judge Model Name", "markdown", displayed_by_default=False),
-    "u_math_acc": Field("U-MATH Acc", "rate", never_hidden=True, tags=["u_math"]),
-    "u_math_text_acc": Field("U-MATH Text Acc", "rate", tags=["u_math", "text"]),
-    "u_math_visual_acc": Field("U-MATH Visual Acc", "rate", tags=["u_math", "visual"]),
+    "u_math_acc": Field("U-MATH Acc", "rate", never_hidden=True, tags=["default"]),
+    "u_math_text_acc": Field("U-MATH Text Acc", "rate", tags=["default", "text"]),
+    "u_math_visual_acc": Field("U-MATH Visual Acc", "rate", tags=["default", "visual"]),
     "differential_calc_acc": Field("Diff Calc Acc", "rate", displayed_by_default=False, tags=["subjects"]),
     "differential_calc_text_acc": Field("Diff Calc Text Acc", "rate", displayed_by_default=False, tags=["text"]),
     "differential_calc_visual_acc": Field(
@@ -208,11 +220,11 @@ MU_MATH_COLUMNS_DICT = {
     "rank": Field("Rank", "number", never_hidden=True),
     **MODEL_COLUMNS_DICT,
     "extract_model_name": Field("Extract Model Name", "markdown", displayed_by_default=False),
-    "mu_math_f1": Field("μ-MATH F1", "rate", never_hidden=True, tags=["mu_math", "splits"]),
-    "mu_math_tpr": Field("μ-MATH TPR", "rate", displayed_by_default=False, tags=["mu_math"]),
-    "mu_math_tnr": Field("μ-MATH TNR", "rate", displayed_by_default=False, tags=["mu_math"]),
-    "mu_math_ppv": Field("μ-MATH PPV", "rate", displayed_by_default=False, tags=["mu_math"]),
-    "mu_math_npv": Field("μ-MATH NPV", "rate", displayed_by_default=False, tags=["mu_math"]),
+    "mu_math_f1": Field("μ-MATH F1", "rate", never_hidden=True, tags=["default", "splits"]),
+    "mu_math_tpr": Field("μ-MATH TPR", "rate", displayed_by_default=False, tags=["default"]),
+    "mu_math_tnr": Field("μ-MATH TNR", "rate", displayed_by_default=False, tags=["default"]),
+    "mu_math_ppv": Field("μ-MATH PPV", "rate", displayed_by_default=False, tags=["default"]),
+    "mu_math_npv": Field("μ-MATH NPV", "rate", displayed_by_default=False, tags=["default"]),
     "GPT-4o_f1": Field("GPT-4o Subset F1", "rate", tags=["splits"]),
     "GPT-4o_tpr": Field("GPT-4o Subset TPR", "rate", displayed_by_default=False),
     "GPT-4o_tnr": Field("GPT-4o Subset TNR", "rate", displayed_by_default=False),
@@ -243,7 +255,7 @@ U_MATH_AND_MU_MATH_COLUMNS_DICT = {
     "u_math_visual_acc": Field("U-MATH Visual Acc", "rate", displayed_by_default=False, tags=["u_math"]),
     "judge_model_name": Field("Judge Model Name", "markdown", displayed_by_default=False),
     "extract_model_name": Field("Extract Model Name", "markdown", displayed_by_default=False),
-    "mu_math_f1": Field("μ-MATH F1", "rate", tags=["main", "u_math", "mu_math"]),
+    "mu_math_f1": Field("μ-MATH F1", "rate", tags=["main", "u_math", "default"]),
     "mu_math_tpr": Field("μ-MATH TPR", "rate", displayed_by_default=False, tags=["mu_math"]),
     "mu_math_tnr": Field("μ-MATH TNR", "rate", displayed_by_default=False, tags=["mu_math"]),
     "mu_math_ppv": Field("μ-MATH PPV", "rate", displayed_by_default=False, tags=["mu_math"]),