Final commit for llama based model

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,6 +1,8 @@
 # Project Specific
+# added git lfs for models 
 viz_env/
-
+.DS_Store
+models/gemma-2-2b
 test.py
 test_123.py
 # Byte-compiled / optimized / DLL files

app.py

@@ -13,28 +13,19 @@ def process_input(input_type, input_value, model_name):
     tokenizer = AutoTokenizer.from_pretrained(model_path+model_name)
     
     if input_type == "Text":
+        character_count = len(input_value)
         # Tokenize the text
-        tokens = tokenizer.tokenize(input_value)
-        token_ids = tokenizer.encode(input_value)
-        
-        # Create output strings
-        # tokens_str = [f"{i+1}. {token}" for i, token in enumerate(tokens)]
-        # token_ids_str = " ".join(map(str, token_ids))
-        
-        return f"Total tokens: {len(tokens)}", tokens, token_ids
+        token_ids = tokenizer.encode(input_value,add_special_tokens=True)
+        tokens = tokenizer.convert_ids_to_tokens(token_ids)
+        return len(tokens),character_count, tokens, token_ids
     
     elif input_type == "Token IDs":
         try:
             token_ids = ast.literal_eval(input_value)
-            # Convert string of token IDs to list of integers
-            # token_ids = list(map(int, input_value.split()))
             # Convert token IDs back to text
-            text = tokenizer.decode(token_ids)
-            # print("The decoded text",text)
-            # Tokenize the text to get individual tokens
-            
+            text = tokenizer.decode(token_ids)            
             # Create output strings
-            return f"Total tokens: {len(token_ids)}", text, input_value
+            return len(token_ids),len(token_ids), text, input_value,
         except ValueError:
             return "Error", "Invalid input. Please enter space-separated integers for Token IDs.", ""
 
@@ -48,13 +39,14 @@ iface = gr.Interface(
     ],
     outputs=[
         gr.Textbox(label="Token Count"),
+        gr.Textbox(label="Character Count"),
         gr.Textbox(label="Tokens", lines=10),
         gr.Textbox(label="Token IDS", lines=5)
     ],
-    title="LLM Tokenization and Token ID Converter",
-    description="Enter text or token IDs and select a model to see the conversion results."
+    title="LLM Tokenization - Convert Text to tokens and vice versa!",
+    description="Enter text or token IDs and select a model to see the results."
 )
 
 if __name__ == "__main__":
     iface.queue()
-    iface.launch()
+    iface.launch()