| { | |
| "model_type": "custom", | |
| "architectures": ["APE"], | |
| "bias" : "False", | |
| "use_rope" : "True", | |
| "use_ape" : "True", | |
| "use_alibi" : "False", | |
| "n_layer": 6, | |
| "n_head": 6, | |
| "n_embd": 384, | |
| "block_size": 64, | |
| "vocab_size": 50304, | |
| "rope_base": 10000.0, | |
| "dropout" : 0 | |
| } |