| run_name: OLMo-1B-as_fm3_omi2 | |
| seed: 6198 | |
| epoch: null | |
| dry_run: false | |
| model: | |
| d_model: 2048 | |
| n_heads: 16 | |
| n_kv_heads: null | |
| clip_qkv: null | |
| n_layers: 16 | |
| mlp_ratio: 8 | |
| mlp_hidden_size: null | |
| activation_type: swiglu | |
| block_type: sequential | |
| block_group_size: 1 | |
| alibi: false | |
| alibi_bias_max: 8.0 | |
| rope: true | |
| rope_full_precision: true | |
| rope_theta: 10000 | |
| flash_attention: false | |
| attention_dropout: 0.0 | |
| multi_query_attention: false | |
| attention_layer_norm: false | |
| residual_dropout: 0.0 | |
| embedding_dropout: 0.0 | |
| embedding_layer_norm: false | |
| layer_norm_type: default | |
| layer_norm_with_affine: false | |
| layer_norm_eps: 1.0e-05 | |
| attention_layer_norm_with_affine: false | |
| max_sequence_length: 2048 | |
| include_bias: false | |
| bias_for_layer_norm: false | |
| scale_logits: false | |
| vocab_size: 32000 | |
| embedding_size: 32000 | |
| weight_tying: true | |
| eos_token_id: 0 | |
| pad_token_id: 1 | |
| init_device: cuda | |
| init_fn: normal | |
| init_std: 0.02 | |
| init_cutoff_factor: 3.0 | |
| precision: amp_bf16 | |
| scale_emb_init: false | |
| emb_init_std: null | |
| norm_after: false | |
| optimizer: | |
| name: adamw | |
| learning_rate: 0.0005 | |
| weight_decay: 0.1 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| eps: 1.0e-08 | |
| no_decay_norm_and_bias: null | |
| selective_updates: false | |
| decay_norm_and_bias: true | |
| decay_embeddings: true | |
| metrics_log_interval: 10 | |
| record_update_metrics: false | |
| scheduler: | |
| name: cosine_with_warmup | |
| units: steps | |
| t_warmup: 2000 | |
| t_max: null | |
| alpha_f: 0.1 | |
| grad_clip_warmup_steps: null | |
| grad_clip_warmup_factor: null | |
| warmup_min_lr: 0.0 | |
| data: | |
| paths: | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct1-tokenized/00000_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00000_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00001_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00002_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00003_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00004_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00005_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00006_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00007_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00008_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00009_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00010_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00011_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00012_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00013_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00014_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00015_00000_doc_shuffled.ds | |
| - data_token/as_fm3_omi2/tinygsm-tokenized/00016_00000_doc_shuffled.ds | |
| memmap_dtype: uint16 | |
| datasets: null | |
| label_mask_paths: null | |
| pad_direction: right | |
| generate_attention_mask: false | |
| generate_doc_lengths: false | |
| num_workers: 32 | |
| drop_last: true | |
| pin_memory: true | |
| prefetch_factor: 8 | |
| persistent_workers: true | |
| timeout: 0 | |
| seed: null | |
| instance_filter: null | |
| custom_dataset: null | |
| restore_dataloader: true | |
| fast_forward_batches: null | |
| evaluators: [] | |
| eval_interval: 5000 | |
| tokenizer: | |
| identifier: meta-llama/Llama-2-7b-hf | |
| truncate_direction: right | |
| save_folder: checkpoints/OLMo-1B-as_fm3_omi2 | |
| remote_save_folder: null | |
| canceled_check_interval: 6000 | |
| save_interval: 3000 | |
| save_interval_unsharded: 3000 | |
| save_interval_ephemeral: null | |
| save_num_checkpoints_to_keep: -1 | |
| save_num_unsharded_checkpoints_to_keep: -1 | |
| save_overwrite: true | |
| force_save_unsharded: false | |
| no_pre_train_checkpoint: false | |
| load_path: null | |
| load_path_sharded_checkpointer: null | |
| try_load_latest_save: false | |
| reset_optimizer_state: false | |
| reset_trainer_state: false | |
| sharded_checkpointer: torch_legacy | |
| new_style_checkpoints: null | |
| max_duration: 1ep | |
| global_train_batch_size: 512 | |
| device_train_batch_size: 128 | |
| device_train_microbatch_size: 16 | |
| device_eval_batch_size: 16 | |
| eval_subset_num_batches: -1 | |
| eval_on_load: false | |
| device_train_grad_accum: 8 | |
| max_grad_norm: 1.0 | |
| max_grad_norm_ratio: null | |
| precision: amp_bf16 | |
| wandb: | |
| project: olmo-debug | |
| entity: null | |
| group: null | |
| name: OLMo-1B-as_fm3_omi2 | |
| tags: | |
| - watching | |
| log_artifacts: false | |
| rank_zero_only: true | |
| log_interval: 1 | |
| speed_monitor: | |
| window_size: 20 | |
| gpu_flops_available: null | |
| console_log_interval: 1 | |
| gen1_gc_interval: 1 | |
| compile: null | |
| distributed_strategy: fsdp | |
| fsdp: | |
| use_orig_params: true | |
| sharding_strategy: FULL_SHARD | |
| wrapping_strategy: null | |
| precision: mixed | |
| hybrid_sharding_num_model_replicas: null | |
| ddp: | |
| grad_sync_mode: batch | |
| find_unused_params: false | |
| single: | |
| device: auto | |
| softmax_auxiliary_loss: false | |
| auxiliary_loss_multiplier: 0.0001 | |
| time_limit: null | |
| extra_steps_after_cancel: 10 | |
| early_stopping_factor: null | |
| save_data_indices: true | |
| python_profiling: false | |
| torch_profiling: false | |
| stop_at: null | |
| stop_after: null | |
| activation_checkpointing: null | |
| fused_loss: null | |
| hf_datasets_cache_dir: null | |
| module_outputs_save_steps: null | |