change to bf16

app.py

@@ -18,7 +18,7 @@ model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
                                              token = os.environ['TOKEN'],
                                              torch_dtype=torch.bfloat16
                                              )
-model = PeftModel.from_pretrained(model,MODEL_NAME)
+model = PeftModel.from_pretrained(model,MODEL_NAME,torch_dtype=torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 if tokenizer.pad_token is None:
     tokenizer.pad_token = tokenizer.eos_token
@@ -55,9 +55,9 @@ model_response = gr.Textbox(
     rtl=True,
 )
 
-temperature = gr.Slider(minimum=0.0,maximum=1.0,value=0.7, label="Temperature")  # Reduced default
-top_k = gr.Slider(1, 10000, value=50, label="Top-k")  # Reduced default
-repetition_penalty = gr.Slider(0.0, 100.0, value=1.2, label="Repetition Penalty")  # Reduced default
+temperature = gr.Slider(minimum=0.01,maximum=1.0,value=0.7, label="Temperature")  # Reduced default
+top_k = gr.Slider(1, 10000, value=10, label="Top-k")  # Reduced default
+repetition_penalty = gr.Slider(0.1, 100.0, value=1.2, label="Repetition Penalty")  # Reduced default
 
 demo=gr.Interface(
     fn=generate,