Update app.py

app.py

@@ -17,28 +17,31 @@ tokenizer_names = [
 # Load all the tokenizers
 tokenizers = {name: AutoTokenizer.from_pretrained(name) for name in tokenizer_names}
 
-def generate_responses(prompt):
+def generate_responses(prompt, selected_tokenizers):
     responses = {}
-    for name, tokenizer in tokenizers.items():
-        try:
-            inputs = tokenizer(prompt, return_tensors="pt")
-            outputs = model.generate(**inputs)
-            response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-            responses[name] = response
-        except Exception as e:
-            responses[name] = f"Error: {str(e)}"
+    for name in selected_tokenizers:
+        tokenizer = tokenizers.get(name)
+        if tokenizer:
+            try:
+                inputs = tokenizer(prompt, return_tensors="pt")
+                outputs = model.generate(**inputs)
+                response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+                responses[name] = response
+            except Exception as e:
+                responses[name] = f"Error: {str(e)}"
     return responses
 
-# Gradio interface setup
+# Gradio interface setup with checkboxes for tokenizers
 interface = gr.Interface(
     fn=generate_responses,
-    inputs=gr.Textbox(lines=2, placeholder="Enter your prompt here..."),
+    inputs=[
+        gr.Textbox(lines=2, placeholder="Enter your prompt here..."),
+        gr.CheckboxGroup(choices=tokenizer_names, label="Select tokenizers to use")
+    ],
     outputs=gr.JSON(),
     title="Tokenizer Comparison",
     description="Compare model outputs with different tokenizers"
 )
 
 # Launch the Gradio interface
-interface.launch()
-
-
+interface.launch()