Update app.py

app.py

@@ -18,22 +18,27 @@ import hmac
 import hashlib
 from uuid import uuid4
 from datetime import datetime
+from huggingface_hub import CommitScheduler, Repository
+from pathlib import Path
 
 load_dotenv()
 
 st.set_page_config(page_title="LLM Healthcare Benchmarking", layout="wide")
 
-
 WRITE_LOCK = threading.Lock()
-DATA_DIR = "data"
-RESULTS_FILE = os.path.join(DATA_DIR, "results.csv")
-
-if not os.path.exists(DATA_DIR):
-    os.makedirs(DATA_DIR)
-    st.success(f"Created `{DATA_DIR}` directory.")
-else:
-    st.info(f"`{DATA_DIR}` directory already exists.")
-
+DATA_DIR = Path("data")
+DATA_DIR.mkdir(exist_ok=True)
+RESULTS_FILE = DATA_DIR / "results.csv"
+
+
+scheduler = CommitScheduler(
+    repo_id=os.getenv("HF_REPO_ID"),       
+    repo_type="dataset",
+    folder_path=DATA_DIR,
+    path_in_repo="data",                
+    every=10,                               
+    token=os.getenv("HF_TOKEN")           
+)
 
 def initialize_session_state():
     if 'api_configured' not in st.session_state:
@@ -54,8 +59,8 @@ def initialize_session_state():
         st.session_state.last_evaluated_dataset = None
 
 initialize_session_state()
-def setup_api_clients():
 
+def setup_api_clients():
     with st.sidebar:
         st.title("API Configuration")
         
@@ -66,8 +71,11 @@ def setup_api_clients():
             password = st.text_input("Password", type="password")
             
             if st.button("Verify Credentials"):
-                if (hmac.compare_digest(username, os.environ.get("STREAMLIT_USERNAME", "")) and 
-                    hmac.compare_digest(password, os.environ.get("STREAMLIT_PASSWORD", ""))):
+                stored_username = os.getenv("STREAMLIT_USERNAME", "")
+                stored_password = os.getenv("STREAMLIT_PASSWORD", "")
+                
+                if (hmac.compare_digest(username, stored_username) and 
+                    hmac.compare_digest(password, stored_password)):
                     try:
                         st.session_state.togetherai_client = OpenAI(
                             api_key=os.getenv('TOGETHERAI_API_KEY'),
@@ -79,7 +87,7 @@ def setup_api_clients():
                         st.session_state.anthropic_client = Anthropic(
                             api_key=os.getenv('ANTHROPIC_API_KEY')
                         )
-                        genai.configure(api_key=os.environ["GEMINI_API_KEY"])
+                        genai.configure(api_key=os.getenv("GEMINI_API_KEY"))
                         
                         st.session_state.api_configured = True
                         st.success("Successfully configured the API clients with stored keys!")
@@ -117,6 +125,10 @@ def setup_api_clients():
                     st.session_state.api_configured = False
 
 setup_api_clients()
+
+
+scheduler.start()
+
 MAX_CONCURRENT_CALLS = 5
 semaphore = threading.Semaphore(MAX_CONCURRENT_CALLS)
 
@@ -191,6 +203,7 @@ def get_model_response(question, options, prompt_template, model_name, clients):
                 )
                 response_text = chat_session.send_message(prompt).text
 
+            # Extract JSON from response
             json_match = re.search(r'\{.*\}', response_text, re.DOTALL)
             if not json_match:
                 return f"Error: Invalid response format", response_text
@@ -206,6 +219,7 @@ def get_model_response(question, options, prompt_template, model_name, clients):
         except Exception as e:
             return f"Error: {str(e)}", str(e)
 
+
 def evaluate_response(model_response, correct_answer):
     if model_response.startswith("Error:"):
         return False
@@ -233,10 +247,15 @@ def process_single_evaluation(question, prompt_template, model_name, clients, la
         'explanation': question['explanation'],
         'timestamp': datetime.utcnow().isoformat()
     }
+    
     with WRITE_LOCK:
-        file_exists = os.path.isfile(RESULTS_FILE)
-        with open(RESULTS_FILE, 'a', encoding='utf-8', newline='') as f:
-            pd.DataFrame([result]).to_csv(f, header=not file_exists, index=False)
+        if RESULTS_FILE.exists():
+            existing_df = pd.read_csv(RESULTS_FILE)
+            updated_df = existing_df.append(result, ignore_index=True)
+        else:
+            updated_df = pd.DataFrame([result])
+        
+        updated_df.to_csv(RESULTS_FILE, index=False)
     
     return result
 
@@ -245,8 +264,7 @@ def process_evaluations_concurrently(questions, prompt_template, models_to_evalu
     total_iterations = len(models_to_evaluate) * len(questions)
     current_iteration = 0
 
-    # Load existing results to avoid re-processing
-    if os.path.exists(RESULTS_FILE):
+    if RESULTS_FILE.exists():
         existing_df = pd.read_csv(RESULTS_FILE)
         completed = set(zip(existing_df['model'], existing_df['question']))
     else:
@@ -283,7 +301,7 @@ def main():
     if 'all_results' not in st.session_state:
         st.session_state.all_results = {}
         st.session_state.last_evaluated_dataset = None
-    if os.path.exists(RESULTS_FILE):
+    if RESULTS_FILE.exists():
         existing_df = pd.read_csv(RESULTS_FILE)
         all_results = {}
         for _, row in existing_df.iterrows():
@@ -302,18 +320,14 @@ def main():
 
     with st.sidebar:
         if st.button("Reset Results"):
-            if os.path.exists(RESULTS_FILE):
-                os.remove(RESULTS_FILE)
-                for file in os.listdir(DATA_DIR):
-                    file_path = os.path.join(DATA_DIR, file)
-                    try:
-                        if os.path.isfile(file_path):
-                            os.unlink(file_path)
-                    except Exception as e:
-                        st.error(f"Error deleting file {file_path}: {e}")
-                st.session_state.all_results = {}
-                st.session_state.last_evaluated_dataset = None
-                st.success("Results have been reset.")
+            if RESULTS_FILE.exists():
+                try:
+                    RESULTS_FILE.unlink()
+                    st.session_state.all_results = {}
+                    st.session_state.last_evaluated_dataset = None
+                    st.success("Results have been reset.")
+                except Exception as e:
+                    st.error(f"Error deleting file: {str(e)}")
             else:
                 st.info("No results to reset.")
 
@@ -334,7 +348,6 @@ def main():
 
     models_to_evaluate = selected_models
 
-
     default_prompt = '''You are a medical AI assistant. Please answer the following multiple choice question.
 Question: {question}
 
@@ -358,7 +371,7 @@ Important:
 - Only the "answer" field will be used for evaluation
 - Ensure your response is in valid JSON format'''
 
-
+    # Customize Prompt Template
     col1, col2 = st.columns([2, 1])
     with col1:
         prompt_template = st.text_area(
@@ -375,74 +388,90 @@ Important:
         - `{options}`: The multiple choice options
         """)
 
+    # Load Dataset
+    if st.session_state.api_configured:
+        with st.spinner("Loading dataset..."):
+            questions = load_dataset_by_name(selected_dataset)
+    else:
+        st.warning("Please configure the API keys in the sidebar to load datasets and proceed.")
+        questions = []
 
-    with st.spinner("Loading dataset..."):
-        questions = load_dataset_by_name(selected_dataset)
-
-
-    subjects = sorted(list(set(q['subject_name'] for q in questions)))
-    selected_subject = st.selectbox("Filter by subject", ["All"] + subjects)
-    
-    if selected_subject != "All":
-        questions = [q for q in questions if q['subject_name'] == selected_subject]
-
-
-    num_questions = st.number_input("Number of questions to evaluate", min_value=1, max_value=len(questions), value=1, step=1)
-
-
-    if st.button("Start Evaluation"):
-        with st.spinner("Starting evaluation..."):
-            selected_questions = questions[:num_questions]
-            
-
-            clients = {
-                "togetherai": st.session_state["togetherai_client"],
-                "openai": st.session_state["openai_client"],
-                "anthropic": st.session_state["anthropic_client"]
-            }
-            
-
-            last_evaluated_dataset = st.session_state.last_evaluated_dataset if st.session_state.last_evaluated_dataset else selected_dataset
-
-            progress_container = st.container()
-            progress_bar = progress_container.progress(0)
-            status_text = progress_container.empty()
-            
-            def update_progress(current, total):
-                progress = current / total
-                progress_bar.progress(progress)
-                status_text.text(f"Progress: {current}/{total} evaluations completed")
-
-            results = process_evaluations_concurrently(
-                selected_questions,
-                prompt_template,
-                models_to_evaluate,
-                update_progress,
-                clients,
-                last_evaluated_dataset
-            )
+    # Filter by Subject
+    if questions:
+        subjects = sorted(list(set(q['subject_name'] for q in questions)))
+        selected_subject = st.selectbox("Filter by subject", ["All"] + subjects)
         
-        all_results = st.session_state.all_results.copy()
-        for result in results:
-            model = result.pop('model')
-            if model not in all_results:
-                all_results[model] = []
-            all_results[model].append(result)
-        
-        st.session_state.all_results = all_results
-        st.session_state.last_evaluated_dataset = selected_dataset
-
-        if st.session_state.detailed_model is None and all_results:
-            st.session_state.detailed_model = list(all_results.keys())[0]
-        if st.session_state.detailed_dataset is None:
-            st.session_state.detailed_dataset = selected_dataset
+        if selected_subject != "All":
+            questions = [q for q in questions if q['subject_name'] == selected_subject]
+
+        # Number of Questions to Evaluate
+        num_questions = st.number_input(
+            "Number of questions to evaluate", 
+            min_value=1, 
+            max_value=len(questions), 
+            value=min(10, len(questions)), 
+            step=1
+        )
 
-        st.success("Evaluation completed!")
-        st.rerun()
+        # Start Evaluation Button
+        if st.button("Start Evaluation"):
+            if not models_to_evaluate:
+                st.error("Please select at least one model to evaluate.")
+            else:
+                with st.spinner("Starting evaluation..."):
+                    selected_questions = questions[:num_questions]
+                    
+                    clients = {
+                        "togetherai": st.session_state["togetherai_client"],
+                        "openai": st.session_state["openai_client"],
+                        "anthropic": st.session_state["anthropic_client"]
+                    }
+                    
+                    last_evaluated_dataset = st.session_state.last_evaluated_dataset if st.session_state.last_evaluated_dataset else selected_dataset
 
+                    progress_container = st.container()
+                    progress_bar = progress_container.progress(0)
+                    status_text = progress_container.empty()
+                    
+                    def update_progress(current, total):
+                        progress = current / total
+                        progress_bar.progress(progress)
+                        status_text.text(f"Progress: {current}/{total} evaluations completed")
+
+                    results = process_evaluations_concurrently(
+                        selected_questions,
+                        prompt_template,
+                        models_to_evaluate,
+                        update_progress,
+                        clients,
+                        last_evaluated_dataset
+                    )
+                
+                # Update Session State with New Results
+                all_results = st.session_state.all_results.copy()
+                for result in results:
+                    model = result.pop('model')
+                    if model not in all_results:
+                        all_results[model] = []
+                    all_results[model].append(result)
+                
+                st.session_state.all_results = all_results
+                st.session_state.last_evaluated_dataset = selected_dataset
+
+                # Set Default Detailed Model and Dataset if Not Set
+                if st.session_state.detailed_model is None and all_results:
+                    st.session_state.detailed_model = list(all_results.keys())[0]
+                if st.session_state.detailed_dataset is None:
+                    st.session_state.detailed_dataset = selected_dataset
+
+                st.success("Evaluation completed!")
+                st.experimental_rerun()
+
+    # Display Evaluation Results
     if st.session_state.all_results:
         st.subheader("Evaluation Results")
-        
+        model_metrics = {}
+
         for model_name, results in st.session_state.all_results.items():
             df = pd.DataFrame(results)
             metrics = {
@@ -450,27 +479,28 @@ Important:
             }
             model_metrics[model_name] = metrics
 
-        metrics_df = pd.DataFrame(model_metrics).T
+        metrics_df = pd.DataFrame(model_metrics).T.reset_index().rename(columns={'index': 'Model'})
 
         st.subheader("Model Performance Comparison")
         accuracy_chart = alt.Chart(
-            metrics_df.reset_index().melt(id_vars=['index'], value_vars=['Accuracy'])
+            metrics_df
         ).mark_bar().encode(
-            x=alt.X('index:N', title=None, axis=None), 
-            y=alt.Y('value:Q', title='Accuracy', scale=alt.Scale(domain=[0, 1])),
-            color=alt.Color('index:N', scale=alt.Scale(scheme='blues')), 
-            tooltip=['index:N', 'value:Q']
+            x=alt.X('Model:N', title=None),
+            y=alt.Y('Accuracy:Q', title='Accuracy', scale=alt.Scale(domain=[0, 1])),
+            color=alt.Color('Model:N', scale=alt.Scale(scheme='blues')),
+            tooltip=['Model:N', 'Accuracy:Q']
         ).properties(
             height=300,
             title={
                 "text": "Model Accuracy",
-                "baseline": "bottom",
-                "orient": "bottom",  
-                "dy": 20              
+                "anchor": "middle",
+                "fontSize": 20
             }
-        )
+        ).interactive()
 
         st.altair_chart(accuracy_chart, use_container_width=True)
+
+    # Display Detailed Results
     if st.session_state.all_results:
         st.subheader("Detailed Results")
         
@@ -494,12 +524,12 @@ Important:
         with col2:
             selected_dataset_details = st.selectbox(
                 "Select dataset",
-                options=[st.session_state.last_evaluated_dataset], 
+                options=[st.session_state.last_evaluated_dataset] if st.session_state.last_evaluated_dataset else [],
                 key="dataset_select",
                 on_change=update_dataset
             )
 
-        if selected_model_details in st.session_state.all_results:
+        if selected_model_details and selected_model_details in st.session_state.all_results:
             results = st.session_state.all_results[selected_model_details]
             df = pd.DataFrame(results)
             accuracy = df['is_correct'].mean()
@@ -537,21 +567,17 @@ Important:
 
         st.markdown("---")
         st.subheader("Download Results")
-        all_data = []
-        for model_name, results in st.session_state.all_results.items():
-            for result in results:
-                row = result.copy()
-                all_data.append(row)
-
-        complete_df = pd.DataFrame(all_data)
-        csv = complete_df.to_csv(index=False)
-        st.download_button(
-            label="Download All Results as CSV",
-            data=csv,
-            file_name=f"all_models_{st.session_state.last_evaluated_dataset}_results.csv",
-            mime="text/csv",
-            key="download_all_results"
-        )
+        if RESULTS_FILE.exists():
+            csv_data = RESULTS_FILE.read_text(encoding='utf-8')
+            st.download_button(
+                label="Download All Results as CSV",
+                data=csv_data,
+                file_name=f"all_models_{st.session_state.last_evaluated_dataset}_results.csv",
+                mime="text/csv", 
+                key="download_all_results"
+            )
+        else:
+            st.info("No data available to download.")
 
 if __name__ == "__main__":
     main()