Delete files models/OLMo-1B/config.yaml with huggingface_hub

models/OLMo-1B/config.yaml

@@ -1,413 +0,0 @@
-run_name: OLMo-1B-as_fm3_omi2
-seed: 6198
-epoch: null
-dry_run: false
-model:
-  d_model: 2048
-  n_heads: 16
-  n_kv_heads: null
-  clip_qkv: null
-  n_layers: 16
-  mlp_ratio: 8
-  mlp_hidden_size: null
-  activation_type: swiglu
-  block_type: sequential
-  block_group_size: 1
-  alibi: false
-  alibi_bias_max: 8.0
-  rope: true
-  rope_full_precision: true
-  rope_theta: 10000
-  flash_attention: false
-  attention_dropout: 0.0
-  multi_query_attention: false
-  attention_layer_norm: false
-  residual_dropout: 0.0
-  embedding_dropout: 0.0
-  embedding_layer_norm: false
-  layer_norm_type: default
-  layer_norm_with_affine: false
-  layer_norm_eps: 1.0e-05
-  attention_layer_norm_with_affine: false
-  max_sequence_length: 2048
-  include_bias: false
-  bias_for_layer_norm: false
-  scale_logits: false
-  vocab_size: 32000
-  embedding_size: 32000
-  weight_tying: true
-  eos_token_id: 0
-  pad_token_id: 1
-  init_device: cuda
-  init_fn: normal
-  init_std: 0.02
-  init_cutoff_factor: 3.0
-  precision: amp_bf16
-  scale_emb_init: false
-  emb_init_std: null
-  norm_after: false
-optimizer:
-  name: adamw
-  learning_rate: 0.0005
-  weight_decay: 0.1
-  betas:
-  - 0.9
-  - 0.95
-  eps: 1.0e-08
-  no_decay_norm_and_bias: null
-  selective_updates: false
-  decay_norm_and_bias: true
-  decay_embeddings: true
-  metrics_log_interval: 10
-  record_update_metrics: false
-scheduler:
-  name: cosine_with_warmup
-  units: steps
-  t_warmup: 2000
-  t_max: null
-  alpha_f: 0.1
-  grad_clip_warmup_steps: null
-  grad_clip_warmup_factor: null
-  warmup_min_lr: 0.0
-data:
-  paths:
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct1-tokenized/00000_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00000_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00001_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00002_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00003_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00004_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00005_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00006_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00007_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00008_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00009_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00010_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00011_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00012_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00013_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00014_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00015_00000_doc_shuffled.ds
-  - data_token/as_fm3_omi2/tinygsm-tokenized/00016_00000_doc_shuffled.ds
-  memmap_dtype: uint16
-  datasets: null
-  label_mask_paths: null
-  pad_direction: right
-  generate_attention_mask: false
-  generate_doc_lengths: false
-  num_workers: 32
-  drop_last: true
-  pin_memory: true
-  prefetch_factor: 8
-  persistent_workers: true
-  timeout: 0
-  seed: null
-  instance_filter: null
-  custom_dataset: null
-restore_dataloader: true
-fast_forward_batches: null
-evaluators: []
-eval_interval: 5000
-tokenizer:
-  identifier: meta-llama/Llama-2-7b-hf
-  truncate_direction: right
-save_folder: checkpoints/OLMo-1B-as_fm3_omi2
-remote_save_folder: null
-canceled_check_interval: 6000
-save_interval: 3000
-save_interval_unsharded: 3000
-save_interval_ephemeral: null
-save_num_checkpoints_to_keep: -1
-save_num_unsharded_checkpoints_to_keep: -1
-save_overwrite: true
-force_save_unsharded: false
-no_pre_train_checkpoint: false
-load_path: step9000-unsharded
-load_path_sharded_checkpointer: null
-try_load_latest_save: false
-reset_optimizer_state: false
-reset_trainer_state: false
-sharded_checkpointer: torch_legacy
-new_style_checkpoints: null
-max_duration: 1ep
-global_train_batch_size: 512
-device_train_batch_size: 128
-device_train_microbatch_size: 16
-device_eval_batch_size: 16
-eval_subset_num_batches: -1
-eval_on_load: false
-device_train_grad_accum: 8
-max_grad_norm: 1.0
-max_grad_norm_ratio: null
-precision: amp_bf16
-wandb:
-  project: olmo-debug
-  entity: null
-  group: null
-  name: OLMo-1B-as_fm3_omi2
-  tags:
-  - watching
-  log_artifacts: false
-  rank_zero_only: true
-  log_interval: 1
-speed_monitor:
-  window_size: 20
-  gpu_flops_available: null
-console_log_interval: 1
-gen1_gc_interval: 1
-compile: null
-distributed_strategy: fsdp
-fsdp:
-  use_orig_params: true
-  sharding_strategy: FULL_SHARD
-  wrapping_strategy: null
-  precision: mixed
-  hybrid_sharding_num_model_replicas: null
-ddp:
-  grad_sync_mode: batch
-  find_unused_params: false
-single:
-  device: auto
-softmax_auxiliary_loss: false
-auxiliary_loss_multiplier: 0.0001
-time_limit: null
-extra_steps_after_cancel: 10
-early_stopping_factor: null
-save_data_indices: true
-python_profiling: false
-torch_profiling: false
-stop_at: null
-stop_after: null
-activation_checkpointing: null
-fused_loss: null
-hf_datasets_cache_dir: null
-module_outputs_save_steps: null