Update app.py

app.py

@@ -205,6 +205,7 @@ def create_strategic_masks(text, tokenizer, strategy="content_words"):
 def symbolic_classification_analysis(text, selected_roles, masking_strategy="content_words", num_predictions=5):
     """
     Perform symbolic classification analysis using MLM prediction
+    FIXED: Now tests what the model actually learned
     """
     if not selected_roles:
         selected_roles = list(symbolic_token_ids.keys())
@@ -213,86 +214,192 @@ def symbolic_classification_analysis(text, selected_roles, masking_strategy="con
         return "Please enter some text to analyze.", "", 0
     
     try:
-        # Create strategically masked input
-        masked_input_ids, attention_mask, original_tokens, mask_positions = create_strategic_masks(
-            text, tokenizer, masking_strategy
-        )
-        
-        if not mask_positions:
-            return "No suitable positions found for masking. Try different text or strategy.", "", 0
+        # DETECT if input follows training pattern vs needs conversion
+        if any(role in text for role in symbolic_token_ids.keys()):
+            # Input already has symbolic tokens - test descriptive prediction
+            return test_descriptive_prediction(text, selected_roles, num_predictions)
+        else:
+            # Convert input to training-style format and test
+            return test_with_context_injection(text, selected_roles, num_predictions)
         
-        # Move to device
-        masked_input_ids = masked_input_ids.to("cuda")
-        attention_mask = attention_mask.to("cuda")
+    except Exception as e:
+        error_msg = f"Error during analysis: {str(e)}"
+        print(error_msg)
+        return error_msg, "", 0
+
+
+def test_descriptive_prediction(text, selected_roles, num_predictions):
+    """
+    Test what descriptive words the model predicts after symbolic tokens
+    This matches the actual training objective
+    """
+    # Find positions after symbolic tokens
+    tokens = tokenizer.tokenize(text, add_special_tokens=True)
+    token_ids = tokenizer.convert_tokens_to_ids(tokens)
+    
+    # Find symbolic token positions
+    symbolic_positions = []
+    for i, token in enumerate(tokens):
+        if token in symbolic_token_ids:
+            # Mask the next 1-3 positions after symbolic token
+            for offset in range(1, min(4, len(tokens) - i)):
+                if i + offset < len(tokens) and tokens[i + offset] not in ['[SEP]', '[PAD]']:
+                    symbolic_positions.append({
+                        'mask_pos': i + offset,
+                        'symbolic_token': token,
+                        'original_token': tokens[i + offset]
+                    })
+    
+    if not symbolic_positions:
+        return "No symbolic tokens found in input. Try format like: '<subject> a young woman'", "", 0
+    
+    # Create masked versions and get predictions
+    results = []
+    for pos_info in symbolic_positions[:5]:  # Limit to 5 positions
+        masked_ids = token_ids.copy()
+        masked_ids[pos_info['mask_pos']] = MASK_ID
+        
+        # Get MLM predictions
+        masked_input = torch.tensor([masked_ids]).to("cuda")
+        attention_mask = torch.ones_like(masked_input)
+        
+        with torch.no_grad():
+            outputs = full_model(input_ids=masked_input, attention_mask=attention_mask)
+            logits = outputs.logits[0, pos_info['mask_pos']]  # Logits for masked position
+        
+        # Get top 10 predictions from full vocabulary
+        probs = F.softmax(logits, dim=-1)
+        top_indices = torch.argsort(probs, descending=True)[:num_predictions]
+        
+        predictions = []
+        for idx in top_indices:
+            token_text = tokenizer.convert_ids_to_tokens([idx.item()])[0]
+            prob = probs[idx].item()
+            predictions.append({
+                "token": token_text,
+                "probability": prob
+            })
         
-        # Get symbolic predictions
-        predictions = get_symbolic_predictions(
-            masked_input_ids, attention_mask, mask_positions, selected_roles
+        results.append({
+            "symbolic_context": pos_info['symbolic_token'],
+            "position": pos_info['mask_pos'],
+            "original_token": pos_info['original_token'],
+            "predictions": predictions
+        })
+    
+    # Format results
+    analysis = {
+        "input_text": text,
+        "test_type": "descriptive_prediction",
+        "explanation": "Testing what descriptive words model predicts after symbolic tokens",
+        "results": results
+    }
+    
+    summary_lines = [f"🎯 Testing Descriptive Prediction (what model actually learned)\n"]
+    for result in results:
+        ctx = result["symbolic_context"]
+        orig = result["original_token"] 
+        top_pred = result["predictions"][0]
+        
+        summary_lines.append(
+            f"After {ctx}: '{orig}' → '{top_pred['token']}' ({top_pred['probability']:.4f})"
         )
-        
-        # Build detailed analysis
-        analysis = {
-            "input_text": text,
-            "masking_strategy": masking_strategy,
-            "total_tokens": len(original_tokens),
-            "masked_positions": len(mask_positions),
-            "available_symbolic_roles": len(selected_roles),
-            "analysis_results": []
-        }
-        
-        for pred_data in predictions:
-            pos = pred_data["position"]
-            original_token = original_tokens[pos]
-            
-            # Show top N predictions
-            top_preds = pred_data["predictions"][:num_predictions]
-            
-            position_analysis = {
-                "position": pos,
-                "original_token": original_token,
-                "top_predictions": []
-            }
+    
+    summary = "\n".join(summary_lines)
+    return json.dumps(analysis, indent=2), summary, len(results)
+
+
+def test_with_context_injection(text, selected_roles, num_predictions):
+    """
+    Inject symbolic context and test what descriptive words are predicted
+    """
+    results = []
+    
+    # Test each selected symbolic role as context
+    for role in selected_roles[:3]:  # Limit to 3 roles for speed
+        # Create training-style context
+        context_text = f"{role} {text}"
+        
+        # Tokenize and find good positions to mask
+        tokens = tokenizer.tokenize(context_text, add_special_tokens=True)
+        token_ids = tokenizer.convert_tokens_to_ids(tokens)
+        
+        # Find role position and mask next content word
+        role_pos = None
+        for i, token in enumerate(tokens):
+            if token == role:
+                role_pos = i
+                break
+        
+        if role_pos is None or role_pos + 2 >= len(tokens):
+            continue
             
-            for pred in top_preds:
-                position_analysis["top_predictions"].append({
-                    "symbolic_role": pred["token"],
-                    "probability": f"{pred['probability']:.4f}",
-                    "confidence": "High" if pred["probability"] > 0.3 else "Medium" if pred["probability"] > 0.1 else "Low"
-                })
+        # Mask position after role (skip articles like "a", "the")
+        mask_pos = role_pos + 1
+        skip_words = {'a', 'an', 'the', 'some', 'this', 'that'}
+        while mask_pos < len(tokens) - 1:
+            current_token = tokens[mask_pos].lower()
+            if current_token not in skip_words and len(current_token) > 2:
+                break
+            mask_pos += 1
+        
+        if mask_pos >= len(tokens):
+            continue
             
-            analysis["analysis_results"].append(position_analysis)
-        
-        # Create readable summary
-        summary_lines = []
-        max_prob = 0
-        best_prediction = None
+        # Create masked input
+        masked_ids = token_ids.copy()
+        original_token = tokens[mask_pos]
+        masked_ids[mask_pos] = MASK_ID
         
-        for result in analysis["analysis_results"]:
-            pos = result["position"]
-            orig = result["original_token"]
-            top_pred = result["top_predictions"][0] if result["top_predictions"] else None
-            
-            if top_pred:
-                prob = float(top_pred["probability"])
-                role = top_pred["symbolic_role"]
-                summary_lines.append(
-                    f"Position {pos:2d}: '{orig}' → {role} ({top_pred['probability']}, {top_pred['confidence']})"
-                )
-                
-                if prob > max_prob:
-                    max_prob = prob
-                    best_prediction = f"{role} (confidence: {top_pred['confidence']})"
+        # Get predictions
+        masked_input = torch.tensor([masked_ids]).to("cuda")
+        attention_mask = torch.ones_like(masked_input)
         
-        summary = "\n".join(summary_lines)
-        if best_prediction:
-            summary = f"🎯 Best Match: {best_prediction}\n\n" + summary
+        with torch.no_grad():
+            outputs = full_model(input_ids=masked_input, attention_mask=attention_mask)
+            logits = outputs.logits[0, mask_pos]
         
-        return json.dumps(analysis, indent=2), summary, len(mask_positions)
+        # Get top predictions
+        probs = F.softmax(logits, dim=-1)
+        top_indices = torch.argsort(probs, descending=True)[:num_predictions]
+        
+        predictions = []
+        for idx in top_indices:
+            token_text = tokenizer.convert_ids_to_tokens([idx.item()])[0]
+            prob = probs[idx].item()
+            predictions.append({
+                "token": token_text,
+                "probability": prob
+            })
         
-    except Exception as e:
-        error_msg = f"Error during analysis: {str(e)}"
-        print(error_msg)
-        return error_msg, "", 0
+        results.append({
+            "symbolic_context": role,
+            "position": mask_pos,
+            "original_token": original_token,
+            "context_text": context_text,
+            "predictions": predictions
+        })
+    
+    # Format results
+    analysis = {
+        "input_text": text,
+        "test_type": "context_injection",
+        "explanation": "Injected symbolic tokens and tested descriptive predictions",
+        "results": results
+    }
+    
+    summary_lines = [f"🎯 Testing with Symbolic Context Injection\n"]
+    for result in results:
+        role = result["symbolic_context"]
+        orig = result["original_token"]
+        top_pred = result["predictions"][0]
+        
+        summary_lines.append(
+            f"{role} context: '{orig}' → '{top_pred['token']}' ({top_pred['probability']:.4f})"
+        )
+    
+    summary = "\n".join(summary_lines)
+    return json.dumps(analysis, indent=2), summary, len(results)
 
 
 def create_manual_mask_analysis(text, mask_positions_str, selected_roles):
@@ -361,7 +468,7 @@ def build_interface():
                     txt_input = gr.Textbox(
                         label="Input Text", 
                         lines=4,
-                        placeholder="Enter text to analyze for symbolic role classification..."
+                        placeholder="Try: '<subject> a young woman wearing elegant dress' or just 'young woman wearing dress'"
                     )
                     
                     with gr.Row():
@@ -450,23 +557,31 @@ def build_interface():
                     )
         
         with gr.Tab("Caption Examples"):
-            gr.Markdown("### 🖼️ Test with Realistic Image Caption Patterns")
+            gr.Markdown("### 🖼️ Test with Training-Style Patterns")
+            gr.Markdown("""
+            **The model was trained to predict descriptive words AFTER symbolic tokens.**
+            
+            Test with patterns like:
+            - `<subject> a young woman wearing elegant dress`
+            - `<lighting> soft natural illumination on the scene`
+            - `<emotion> happy expression while posing confidently`
+            """)
             
             example_captions = [
-                "a young woman wearing a blue dress",
-                "the man has short brown hair",  
-                "she is wearing red high heels",
-                "the lighting is soft and natural",
-                "her expression shows happiness",
-                "a person standing in confident pose",
-                "wearing elegant silver jewelry",
-                "the fabric has floral pattern"
+                "<subject> a young woman wearing a blue dress",
+                "<lighting> soft natural illumination in the scene",  
+                "<emotion> happy expression while posing confidently",
+                "<pose> standing gracefully near the window",
+                "<upper_body_clothing> elegant silk blouse with intricate patterns",
+                "<material> luxurious velvet fabric with rich texture",
+                "<accessory> delicate silver jewelry catching the light",
+                "<surface> polished marble floor reflecting ambient glow"
             ]
             
             for caption in example_captions:
                 with gr.Row():
-                    gr.Textbox(value=caption, label="Example Caption", interactive=False, scale=3)
-                    copy_btn = gr.Button("📋 Copy", scale=1)
+                    gr.Textbox(value=caption, label="Training-Style Example", interactive=False, scale=3)
+                    copy_btn = gr.Button("📋 Test This", scale=1)
         
         # Event handlers
         analyze_btn.click(