|
|
#!/bin/bash |
|
|
|
|
|
|
|
|
set -x |
|
|
|
|
|
python ./_infer.py \ |
|
|
model.path=./checkpoints/model \ |
|
|
model.load_param=False \ |
|
|
data.path=./data/parquet/test.parquet \ |
|
|
data.output_path=./model_output/[email protected] \ |
|
|
data.batch_size=32 data.n_samples=1 \ |
|
|
rollout.tensor_model_parallel_size=1 \ |
|
|
rollout.temperature=0.7 rollout.top_p=0.9 rollout.n=1 rollout.do_sample=True \ |
|
|
rollout.prompt_length=1200 rollout.response_length=512 \ |
|
|
rollout.enable_chunked_prefill=True \ |
|
|
+rollout.kv_cache_dtype=fp8_e5m2 \ |
|
|
rollout.max_model_len=1800 \ |
|
|
rollout.max_num_batched_tokens=1800 \ |
|
|
rollout.max_num_seqs=1 \ |
|
|
+model.trust_remote_code=True \ |
|
|
+rollout.kv_cache_block_size=16 \ |
|
|
+rollout.swap_space=16 \ |
|
|
rollout.gpu_memory_utilization=0.7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
